G-24: one composition run per candidate and machine, one test run per level, every reading step timed #244
@@ -0,0 +1,375 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// A composition run's bound (design 47 G §720; A-466n, A-466p, A-466q, A-466s, A-468b, A-469b, A-469s, A-469y, A-470): a run
|
||||
// has one clock, the key `bound.<run key>` of composition-runs with its own time to live composition-step-bound, written at the
|
||||
// build seat's ask-taken of the run's ask and written again at each step-progress of that ask; an ask still waiting in the queue
|
||||
// has no clock but `untaken.<run key>`, written at the run's start with the time to live run-untaken-bound. Either's expiry,
|
||||
// heard as the bus's event, ends the run: its ask dropped through the build seat's drop, the run ended could not run naming the
|
||||
// ask and the bound, so every reader ends could not run with it. While the ask waits for a build node each reader is said waiting
|
||||
// by a wait-held on mesh-wants (`on: build node`), ended `ran` at ask-taken or `superseded` when the run ends untaken.
|
||||
|
||||
// waitOnBuildNode is what a composition run's waiting reader waits on (A-469b, A-469y).
|
||||
const waitOnBuildNode = "build node"
|
||||
|
||||
// dropVerbKey is the Go build seat's drop, asked as a kept call (§119, A-182).
|
||||
const dropVerbKey = "seat:" + nodebuildgo.Seat + "." + nodebuildgo.VerbDrop
|
||||
|
||||
// dropOver asks the Go build seat's drop of a run ask as a kept call, of any holder, which records it (A-469s).
|
||||
func dropOver(call func(key string, body any) (json.RawMessage, error)) func(ask, why string) error {
|
||||
return func(ask, why string) error {
|
||||
raw, err := call(dropVerbKey, nodebuildgo.DropInput{ID: ask, Why: why, By: runAsker})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, said, ok := buscalls.Unwrap(raw); !ok {
|
||||
return fmt.Errorf("%s answered: %s", dropVerbKey, said)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// waitOf is a reader's wait for a build node to take its run's ask (A-469y).
|
||||
func waitOf(run meshwants.CompositionRun, reader meshwants.CompositionReader) meshwants.Wait {
|
||||
w := meshwants.Wait{Step: "pipeline", Owner: meshwants.Seat, Module: reader.Module, Node: run.Machine, On: waitOnBuildNode, Fp: reader.Fingerprint,
|
||||
Words: fmt.Sprintf("the composition step of %s waits for a build node to take the composition run %s of %s (ask %s)", reader.Pipeline,
|
||||
runKeyOf(run.Candidate, run.Machine), run.Machine, run.Ask)}
|
||||
if run.Started != nil {
|
||||
w.Since = *run.Started
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// sayWaits says, for each reader, its wait held or ended (how: ran, superseded); a wait that cannot be said is journaled.
|
||||
func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshwants.CompositionReader, how string) {
|
||||
if r.say == nil {
|
||||
return
|
||||
}
|
||||
for _, c := range readers {
|
||||
w := waitOf(run, c)
|
||||
event, id := meshwants.EventWaitHeld, "composition."+runKeyOf(run.Candidate, run.Machine)+"."+run.Ask+"."+c.ID+".held"
|
||||
if how != "" {
|
||||
w.How, event, id = how, meshwants.EventWaitEnded, strings.TrimSuffix(id, ".held")+".ended"
|
||||
}
|
||||
if err := r.say(event, w, id); err != nil {
|
||||
r.logf("the %s of %s on the composition run of %s is not said: %v", event, c.Pipeline, run.Machine, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n).
|
||||
func (r *compositionRuns) untaken(key, ask string) bool {
|
||||
held, _, standing, err := r.boundOf(runUntakenPrefix + key)
|
||||
return err == nil && standing && held == ask
|
||||
}
|
||||
|
||||
// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written.
|
||||
func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) {
|
||||
raw, rev, err := r.kv.get(boundKey)
|
||||
if err != nil || raw == nil {
|
||||
return "", 0, false, err
|
||||
}
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(raw, &v)
|
||||
return v.Ask, rev, true, nil
|
||||
}
|
||||
|
||||
// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when
|
||||
// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier
|
||||
// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run.
|
||||
func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) {
|
||||
rev, err := r.kv.arm(prefix+key, ask, ttl)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if ok && run.Ended == nil && run.Ask == ask {
|
||||
return true, nil
|
||||
}
|
||||
if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) {
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask
|
||||
// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or
|
||||
// ended, which moves nothing and is journaled (A-470n).
|
||||
func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error {
|
||||
key, run, err := r.runOfAsk(e.Asker, e.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if key == "" {
|
||||
r.late("ask-taken", e.Asker, e.Ask)
|
||||
return nil
|
||||
}
|
||||
armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err)
|
||||
}
|
||||
if !armed {
|
||||
r.late("ask-taken", e.Asker, e.Ask)
|
||||
return nil
|
||||
}
|
||||
if r.untaken(key, e.Ask) {
|
||||
if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
r.sayWaits(run, run.Readers, "ran")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a
|
||||
// late one moves nothing and is journaled (A-470n).
|
||||
func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error {
|
||||
key, _, err := r.runOfAsk(e.Asker, e.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
armed := false
|
||||
if key != "" {
|
||||
if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !armed {
|
||||
r.late("step-progress", e.Asker, e.Ask)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n).
|
||||
func (r *compositionRuns) late(signal, asker, ask string) {
|
||||
if asker == runAsker && ask != "" {
|
||||
r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask)
|
||||
}
|
||||
}
|
||||
|
||||
// runOfAsk is the running run of an ask the assigner put, or none.
|
||||
func (r *compositionRuns) runOfAsk(asker, ask string) (string, meshwants.CompositionRun, error) {
|
||||
if asker != runAsker || ask == "" {
|
||||
return "", meshwants.CompositionRun{}, nil
|
||||
}
|
||||
key, run, err := r.keyOfAsk(ask)
|
||||
if err != nil || key == "" || run.Ended != nil {
|
||||
return "", meshwants.CompositionRun{}, err
|
||||
}
|
||||
return key, run, nil
|
||||
}
|
||||
|
||||
// expired ends the run whose bound key's time to live ended (§720): its ask dropped, the run ended could not run naming the
|
||||
// ask and the bound. A run that ended already, or a key of none, takes nothing.
|
||||
func (r *compositionRuns) expired(boundKey string) {
|
||||
key, bound, setting := "", time.Duration(0), ""
|
||||
switch {
|
||||
case strings.HasPrefix(boundKey, runBoundPrefix):
|
||||
key, bound, setting = strings.TrimPrefix(boundKey, runBoundPrefix), r.set.stepBound, "composition-step-bound"
|
||||
case strings.HasPrefix(boundKey, runUntakenPrefix):
|
||||
key, bound, setting = strings.TrimPrefix(boundKey, runUntakenPrefix), r.set.untaken, "run-untaken-bound"
|
||||
default:
|
||||
return
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended != nil {
|
||||
return
|
||||
}
|
||||
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and
|
||||
// the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced
|
||||
// since, ends nothing.
|
||||
other := runUntakenPrefix
|
||||
if setting == "run-untaken-bound" {
|
||||
other = runBoundPrefix
|
||||
}
|
||||
_, _, armed, err := r.boundOf(boundKey)
|
||||
if err != nil {
|
||||
r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err)
|
||||
return
|
||||
}
|
||||
held, _, standing, err := r.boundOf(other + key)
|
||||
if err != nil {
|
||||
r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err)
|
||||
return
|
||||
}
|
||||
if armed || (standing && held == run.Ask) {
|
||||
r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask)
|
||||
return
|
||||
}
|
||||
said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting)
|
||||
if setting == "run-untaken-bound" {
|
||||
said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting)
|
||||
}
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, said); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
// The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded.
|
||||
if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil {
|
||||
r.logf("the composition run %s past its bound is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted.
|
||||
func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
|
||||
return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask))
|
||||
}
|
||||
|
||||
// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged.
|
||||
func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) {
|
||||
return r.endRunMoving(key, ask, outcome, said, verdict, untaken, true)
|
||||
}
|
||||
|
||||
// endRunMoving is endRunOf, moving the readers a rerun carried or not: a drop says its readers' ends itself (A-470r).
|
||||
func (r *compositionRuns) endRunMoving(key, ask, outcome, said string, verdict map[string]any, untaken, move bool) (bool, error) {
|
||||
// The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own
|
||||
// step still waits on it, so it alone says its own end; each mover waits for the end in the key.
|
||||
if moved := r.moved.Load(); moved != nil && move {
|
||||
if run, _, ok, err := r.read(key); err == nil && ok && run.Ended == nil && run.Ask == ask {
|
||||
(*moved)(run)
|
||||
}
|
||||
}
|
||||
ended, err := r.end(key, ask, outcome, said, verdict)
|
||||
if err != nil || !ended {
|
||||
return ended, err
|
||||
}
|
||||
if untaken {
|
||||
if run, _, ok, err := r.read(key); err == nil && ok {
|
||||
r.sayWaits(run, run.Readers, "superseded")
|
||||
}
|
||||
}
|
||||
if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil {
|
||||
r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at
|
||||
// its start (§719), each reader's end said from the keys alone (A-468). A reader whose step said its end already, carried into a
|
||||
// rerun's run, is taken up only when it is marked `rerun` (A-470l, A-470m); each with the ask of the open run it is taken up from.
|
||||
func (r *compositionRuns) readersOfOpen() ([]openReader, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []openReader
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended != nil {
|
||||
continue
|
||||
}
|
||||
for _, c := range run.Readers {
|
||||
if c.Ended != nil && !c.Rerun {
|
||||
continue // it ended passed already: only a reader marked rerun says its end again
|
||||
}
|
||||
if !seen[c.ID] {
|
||||
seen[c.ID] = true
|
||||
out = append(out, openReader{CompositionReader: c, ask: run.Ask})
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openReader is a reader taken up at the start and the ask of the open run it is taken up from.
|
||||
type openReader struct {
|
||||
meshwants.CompositionReader
|
||||
ask string
|
||||
}
|
||||
|
||||
// keysOf are the keys of the candidate's runs that the reader reads.
|
||||
func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
prefix := keysPart(candidate) + "."
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) || !strings.HasPrefix(key, prefix) {
|
||||
continue
|
||||
}
|
||||
if run, _, ok, err := r.read(key); err == nil && ok && hasReader(run.Readers, id) {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(out) // by machine, as a step names its machines
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// hearRunBounds hears the build seats' ask-taken and step-progress on the assigner's durable consumer (§720, A-466q): each
|
||||
// handed to the runs; one that cannot be written is given again by the bus.
|
||||
func hearRunBounds(hub interface {
|
||||
Handle(string, func(jetstream.Msg)) error
|
||||
}, r *compositionRuns, logf func(string, ...any)) error {
|
||||
if err := hub.Handle(nodebuildgo.SubjectAskTakenAll, func(m jetstream.Msg) {
|
||||
var e nodebuildgo.AskTaken
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no ask-taken, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
ackOrNak(m, r.heardAskTaken(e), logf)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return hub.Handle(nodebuildgo.SubjectStepProgressAll, func(m jetstream.Msg) {
|
||||
var e nodebuildgo.StepProgress
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no step-progress, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
ackOrNak(m, r.heardProgress(e), logf)
|
||||
})
|
||||
}
|
||||
|
||||
func ackOrNak(m jetstream.Msg, err error, logf func(string, ...any)) {
|
||||
if err != nil {
|
||||
logf("%s is not taken, so the bus gives it again: %v", m.Subject(), err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// takeUp says, at the assigner's start, the end of each reader of a run with no `ended` once its runs end (§719, A-468): the
|
||||
// step's goroutine is gone, so its end is judged from the candidate's keys naming it.
|
||||
func (c *compositionSteps) takeUp(ctx context.Context, r *compositionRuns) {
|
||||
readers, err := r.readersOfOpen()
|
||||
if err != nil {
|
||||
c.logf("composition-runs: the runs standing at the start are not read, so their readers are not taken up: %v", err)
|
||||
return
|
||||
}
|
||||
for _, reader := range readers {
|
||||
c.logf("the composition step %s of %s is taken up again by its runs (ask %s)", reader.Pipeline, reader.Module, reader.ID)
|
||||
again := "" // a reader whose step said its end already says it again under the open run's ask (A-470q)
|
||||
if reader.Ended != nil {
|
||||
again = reader.ask
|
||||
}
|
||||
go c.say1(ctx, reader.ID, stepRunOf(reader.CompositionReader), c.resume, again)
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
@@ -18,8 +17,8 @@ import (
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input,
|
||||
// asked of the machine's node-engine by the seat's verb compose-candidate.
|
||||
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
|
||||
// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts.
|
||||
|
||||
// composeCandidateKey is the seat verb's call key, asked of one node.
|
||||
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
|
||||
@@ -80,37 +79,14 @@ type compositionInputs struct {
|
||||
bound time.Duration
|
||||
}
|
||||
|
||||
// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in
|
||||
// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an
|
||||
// error. The run keeps each request built from an input under its run ask (compositionRun.machine).
|
||||
func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) {
|
||||
key, err := candidateKeyOf(run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := c.machines(run.Module, run.Branch)
|
||||
if len(nodes) == 0 {
|
||||
return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch)
|
||||
}
|
||||
// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the
|
||||
// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set.
|
||||
func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput {
|
||||
bound := c.bound
|
||||
if bound <= 0 {
|
||||
bound = nec.AnswerBound
|
||||
}
|
||||
out := make([]nodeInput, len(nodes))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range nodes {
|
||||
if away != nil && away(node) {
|
||||
out[i] = nodeInput{Node: node, Away: true}
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
out[i] = c.one(ctx, bound, node, key)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return out, nil
|
||||
return c.one(ctx, bound, node, candidate)
|
||||
}
|
||||
|
||||
// one asks one node for its input.
|
||||
|
||||
+29
-4
@@ -68,7 +68,7 @@ func TestTheMachinesOfAStep(t *testing.T) {
|
||||
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
|
||||
// silent or unready node says why, and a machine away is not asked.
|
||||
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
// gather asks the machines side by side, so the record of what each was asked is guarded.
|
||||
// the machines are asked side by side, so the record of what each was asked is guarded.
|
||||
var mu sync.Mutex
|
||||
asked := map[string]string{}
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -86,7 +86,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second}
|
||||
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
|
||||
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
|
||||
if err != nil || len(got) != 4 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
@@ -105,7 +105,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
t.Errorf("the machine away is asked or not said away: %+v", got[3])
|
||||
}
|
||||
none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call}
|
||||
if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
|
||||
if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
|
||||
t.Error("a step with no machine is gathered")
|
||||
}
|
||||
}
|
||||
@@ -124,7 +124,7 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
|
||||
return json.RawMessage(answer + `,"file":"` + file + `"}`), nil
|
||||
}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second}
|
||||
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
|
||||
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
@@ -135,3 +135,28 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
|
||||
t.Errorf("g14: %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away
|
||||
// is not asked; no machine is an error.
|
||||
func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) {
|
||||
key, err := candidateKeyOf(run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := c.machines(run.Module, run.Branch)
|
||||
if len(nodes) == 0 {
|
||||
return nil, errors.New("no machine")
|
||||
}
|
||||
out := make([]nodeInput, len(nodes))
|
||||
var wg sync.WaitGroup
|
||||
for i, node := range nodes {
|
||||
if away != nil && away(node) {
|
||||
out[i] = nodeInput{Node: node, Away: true}
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }()
|
||||
}
|
||||
wg.Wait()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -21,12 +21,14 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The run of a composition step (design 47 G §354, §63, §64; A-226, A-238, A-240, A-243, A-252, A-254, A-261): one
|
||||
// composer.Request per machine, depth compose, the one machine and one case `change`, its manifests, verdicts and seats from the
|
||||
// step's catalog revision read through mesh-catalog's `revision`; one run ask per machine of the seat node-build-go, naming no
|
||||
// head, its program the node-engine build's catalog entry with the composer's package; the run's end heard on the build seat's
|
||||
// step-ended of that ask, its answer (a composer.Verdict) read inline from `output`. A machine the watchman said away
|
||||
// (machine-away) is not composed and passes with a warning (§64).
|
||||
// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one
|
||||
// composition run per machine of its candidate (compositionruns.go), and the step that starts a machine's run composes it: one
|
||||
// compose-candidate ask of the node, one composer.Request, depth compose, the one machine and one case `change`, its manifests,
|
||||
// verdicts and seats from the step's catalog revision read through mesh-catalog's `revision`, kept once in `inputs`; one run ask
|
||||
// of the seat node-build-go, naming no head, its program the node-engine build's catalog entry with the composer's package,
|
||||
// chosen once and recorded in the run. The run's end is heard on the build seat's step-ended of that ask, its answer (a
|
||||
// composer.Verdict) read from `output`, and written in the run's key, whose change every reading step hears. A machine the
|
||||
// watchman said away (machine-away) is not composed and passes with a warning (§64).
|
||||
|
||||
const (
|
||||
// runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run
|
||||
@@ -41,42 +43,6 @@ const (
|
||||
seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions
|
||||
)
|
||||
|
||||
// runEnds hands each build seat's step-ended of an ask to the composition run that waits for it (A-240), by the ask's id.
|
||||
type runEnds struct {
|
||||
mu sync.Mutex
|
||||
waits map[string]chan buildstep.StepEnded
|
||||
}
|
||||
|
||||
// expect is the end of the ask id, once heard; called before the ask is put, so no end is missed.
|
||||
func (r *runEnds) expect(id string) <-chan buildstep.StepEnded {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.waits == nil {
|
||||
r.waits = map[string]chan buildstep.StepEnded{}
|
||||
}
|
||||
ch := make(chan buildstep.StepEnded, 1)
|
||||
r.waits[id] = ch
|
||||
return ch
|
||||
}
|
||||
|
||||
// forget stops waiting for the ask id.
|
||||
func (r *runEnds) forget(id string) {
|
||||
r.mu.Lock()
|
||||
delete(r.waits, id)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// heard hands an end to the run that waits for its ask; one nobody waits for is another asker's, left.
|
||||
func (r *runEnds) heard(e buildstep.StepEnded) {
|
||||
r.mu.Lock()
|
||||
ch := r.waits[e.Ask]
|
||||
delete(r.waits, e.Ask)
|
||||
r.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- e
|
||||
}
|
||||
}
|
||||
|
||||
// awayNodes are the machines the watchman said away (machine-away of the seat mesh-watchdogs, §389) and not back since.
|
||||
type awayNodes struct {
|
||||
mu sync.Mutex
|
||||
@@ -109,6 +75,8 @@ func (a *awayNodes) is(machine string) bool {
|
||||
|
||||
// compositionRun runs a composition step.
|
||||
type compositionRun struct {
|
||||
// runs are the composition runs of the state composition-runs (compositionruns.go).
|
||||
runs *compositionRuns
|
||||
inputs *compositionInputs
|
||||
keeper *inputKeeper
|
||||
// call asks a seat verb held once for the mesh (process.Conn.CallKept).
|
||||
@@ -119,8 +87,7 @@ type compositionRun struct {
|
||||
running func(node, module string) (ne.Running, bool)
|
||||
away *awayNodes
|
||||
// put puts a run ask on the build seat's check queue under the assigner's name.
|
||||
put func(ctx context.Context, a buildstep.StepAsk) error
|
||||
ends *runEnds
|
||||
put func(ctx context.Context, a buildstep.StepAsk) error
|
||||
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
|
||||
reads *compositionReads
|
||||
sleep func(ctx context.Context, d time.Duration) error
|
||||
@@ -132,46 +99,81 @@ type compositionRun struct {
|
||||
|
||||
// machineEnd is what one machine came to.
|
||||
type machineEnd struct {
|
||||
failed, couldNot, warning string
|
||||
failed, couldNot string
|
||||
warnings []string
|
||||
// judged is set where the composer judged the machine, composing or not (A-467a).
|
||||
judged bool
|
||||
}
|
||||
|
||||
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
|
||||
// could not be composed or was not judged, else passed with a warning per machine away.
|
||||
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
|
||||
half, err := r.revision(ctx, run)
|
||||
if err != nil {
|
||||
return couldNotRun("the step's catalog revision is not read: " + err.Error())
|
||||
}
|
||||
seats, err := r.seats(ctx, half.Revision.Definitions)
|
||||
// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's
|
||||
// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not
|
||||
// compose, could not run when a machine could not be composed or was not judged, else passed with a warning per machine away.
|
||||
func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
manifests, verdicts, err := caseOf(half)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
nodes := r.inputs.machines(step.Module, step.Branch)
|
||||
if len(nodes) == 0 {
|
||||
return couldNotRun(fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch))
|
||||
}
|
||||
programOf, err := r.programs(ctx, half)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
inputs, err := r.inputs.gather(ctx, run, r.away.is)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
reader := readerOf(id, step)
|
||||
prep := &runPrep{r: r, step: step}
|
||||
var ends []machineEnd
|
||||
var keys []string
|
||||
for _, node := range nodes {
|
||||
if r.away.is(node) {
|
||||
ends = append(ends, machineEnd{warnings: []string{node + " is away (the watchman's machine-away), so it is not composed"}})
|
||||
continue
|
||||
}
|
||||
join := r.runs.join
|
||||
if step.Rerun { // a step asked again (§727)
|
||||
join = r.runs.rejoin
|
||||
}
|
||||
j, err := join(reader, candidate, node, buildstep.NewID())
|
||||
if err != nil {
|
||||
ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()})
|
||||
continue
|
||||
}
|
||||
if j.started {
|
||||
go r.start(ctx, j.key, j.run, prep)
|
||||
}
|
||||
keys = append(keys, j.key)
|
||||
}
|
||||
return r.read(ctx, step.Module, keys, ends)
|
||||
}
|
||||
|
||||
// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the
|
||||
// runs it reads and their asks (§723, §729).
|
||||
func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome {
|
||||
setShared(ctx, r.runs.sharedOf(keys, nil))
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
ends := make([]machineEnd, len(inputs))
|
||||
var wg sync.WaitGroup
|
||||
for i, in := range inputs {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ends[i] = r.machine(ctx, in, buildstep.NewID(), programOf, manifests, verdicts, seats)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
runs := r.runs.await(ctx, keys)
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
|
||||
for _, key := range keys {
|
||||
ends = append(ends, judgeRun(module, runs[key]))
|
||||
}
|
||||
out := outcomeOf(ends)
|
||||
out.Shared = r.runs.sharedOf(keys, runs)
|
||||
return out
|
||||
}
|
||||
|
||||
// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended.
|
||||
func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
keys, err := r.runs.keysOf(candidate, id)
|
||||
if err != nil {
|
||||
return couldNotRun("the step's composition runs are not read: " + err.Error())
|
||||
}
|
||||
return r.read(ctx, step.Module, keys, nil)
|
||||
}
|
||||
|
||||
// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no
|
||||
// machine never passes (A-467a), each machine not judged named with why.
|
||||
func outcomeOf(ends []machineEnd) stepOutcome {
|
||||
var out stepOutcome
|
||||
var failed, couldNot []string
|
||||
judged := 0
|
||||
@@ -181,14 +183,14 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu
|
||||
}
|
||||
failed = appendSome(failed, e.failed)
|
||||
couldNot = appendSome(couldNot, e.couldNot)
|
||||
out.Warnings = appendSome(out.Warnings, e.warning)
|
||||
out.Warnings = append(out.Warnings, e.warnings...)
|
||||
}
|
||||
switch {
|
||||
case len(failed) > 0:
|
||||
out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ")
|
||||
case len(couldNot) > 0:
|
||||
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ")
|
||||
case judged == 0: // a step that judged no machine never passes (A-467a): each machine not judged is named with why
|
||||
case judged == 0:
|
||||
said := "no machine was judged"
|
||||
if len(out.Warnings) == 0 {
|
||||
said += ": the step has no machine"
|
||||
@@ -213,82 +215,187 @@ func couldNotRun(said string) stepOutcome {
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said}
|
||||
}
|
||||
|
||||
// machine composes one machine: its request kept under its ask, the ask put, its end heard and its verdict read.
|
||||
func (r *compositionRun) machine(ctx context.Context, in nodeInput, ask string, programOf func(node string) (buildstep.RunProgram, error),
|
||||
manifests map[string]composer.Manifest, verdicts map[string]json.RawMessage, seats []json.RawMessage) machineEnd {
|
||||
node := in.Node
|
||||
switch {
|
||||
case in.Away:
|
||||
return machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"}
|
||||
case in.Why != "":
|
||||
return machineEnd{couldNot: node + " gave no composition input: " + in.Why}
|
||||
// runPrep is what the step that starts a run reads of its catalog revision, once for every machine it starts: the seats, the
|
||||
// case's manifests and verdicts, and the composer program per machine.
|
||||
type runPrep struct {
|
||||
r *compositionRun
|
||||
step meshwants.StepRun
|
||||
once sync.Once
|
||||
// read once
|
||||
seats []json.RawMessage
|
||||
manifests map[string]composer.Manifest
|
||||
verdicts map[string]json.RawMessage
|
||||
none map[string]string
|
||||
programOf func(node string) (buildstep.RunProgram, string, error)
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *runPrep) get(ctx context.Context) error {
|
||||
p.once.Do(func() {
|
||||
half, err := p.r.revision(ctx, p.step)
|
||||
if err != nil {
|
||||
p.err = errors.New("the step's catalog revision is not read: " + err.Error())
|
||||
return
|
||||
}
|
||||
if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil {
|
||||
return
|
||||
}
|
||||
if p.manifests, p.verdicts, p.none, p.err = caseOf(half); p.err != nil {
|
||||
return
|
||||
}
|
||||
p.programOf, p.err = p.r.programs(ctx, half)
|
||||
})
|
||||
return p.err
|
||||
}
|
||||
|
||||
// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`,
|
||||
// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run.
|
||||
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) {
|
||||
node, ask := run.Machine, run.Ask
|
||||
end := func(said string) {
|
||||
if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil {
|
||||
r.logf("the composition run %s is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
program, err := programOf(node)
|
||||
if err := prep.get(ctx); err != nil {
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
in := r.inputs.of(ctx, node, run.Candidate)
|
||||
if in.Why != "" {
|
||||
end(node + " gave no composition input: " + in.Why)
|
||||
return
|
||||
}
|
||||
program, engineFp, err := prep.programOf(node)
|
||||
if err != nil {
|
||||
return machineEnd{couldNot: node + ": " + err.Error()}
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
|
||||
Cases: []composer.Case{{Name: changeCase, Manifests: manifests, Verdicts: verdicts}}, Seats: seats}
|
||||
Cases: []composer.Case{{Name: changeCase, Manifests: prep.manifests, Verdicts: prep.verdicts, None: prep.none}}, Seats: prep.seats,
|
||||
BlameTries: r.runs.set.blameTries} // the composer's bound of its search for culprits (§721, A-466r)
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return machineEnd{couldNot: node + ": its request cannot be encoded: " + err.Error()}
|
||||
end(node + ": its request cannot be encoded: " + err.Error())
|
||||
return
|
||||
}
|
||||
if _, err := r.keeper.keep(ask, raw); err != nil {
|
||||
digest, err := r.keeper.keep(ask, raw)
|
||||
if err != nil {
|
||||
r.logf("the composition request of %s is refused: %v", node, err)
|
||||
return machineEnd{couldNot: node + ": its request is not kept: " + err.Error()}
|
||||
end(node + ": its request is not kept: " + err.Error())
|
||||
return
|
||||
}
|
||||
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
|
||||
if err != nil {
|
||||
return machineEnd{couldNot: node + ": " + err.Error()}
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: engineFp}
|
||||
c.Input = digest
|
||||
return true
|
||||
}); err != nil {
|
||||
end(node + ": the run's program and input are not written: " + err.Error())
|
||||
return
|
||||
}
|
||||
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
|
||||
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}}
|
||||
ended := r.ends.expect(ask)
|
||||
defer r.ends.forget(ask)
|
||||
if err := r.put(ctx, a); err != nil {
|
||||
return machineEnd{couldNot: node + ": its run ask " + ask + " is not put: " + err.Error()}
|
||||
end(node + ": its run ask " + ask + " is not put: " + err.Error())
|
||||
}
|
||||
var e buildstep.StepEnded
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return machineEnd{couldNot: node + ": the step stopped before its run ask " + ask + " ended"}
|
||||
case e = <-ended:
|
||||
}
|
||||
if e.Outcome != buildstep.OutcomePassed {
|
||||
return machineEnd{couldNot: fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, ask, e.Outcome, e.Node, e.Said)}
|
||||
}
|
||||
answer, err := e.ReadOutput(buildstep.AskGo, r.ask) // the program's last line of standard output (A-254)
|
||||
if err != nil {
|
||||
return machineEnd{couldNot: node + ": " + err.Error()}
|
||||
}
|
||||
return verdictOf(node, answer)
|
||||
}
|
||||
|
||||
// verdictOf reads the composer's verdict of one machine: not composing is failed, naming why; not judged is could not run, naming why (A-467).
|
||||
func verdictOf(node string, answer []byte) machineEnd {
|
||||
// notTried is the problem the composer names a module with that its search for culprits did not try, past the request's
|
||||
// blame_tries (design 47 G §721; the composer's composer.NotTried in mesh-host, compared by its words, A-470b).
|
||||
const notTried = "not tried: the search reached its bound"
|
||||
|
||||
// judgeRun judges one machine of the step of module from the run it read (§721, §722): could not run with the run's words; else
|
||||
// from the machine's composer verdict: not judged is could not run (A-467); a verdict blaming the step's module fails it, or is
|
||||
// could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on
|
||||
// it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the
|
||||
// composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there.
|
||||
func judgeRun(module string, run meshwants.CompositionRun) machineEnd {
|
||||
node := run.Machine
|
||||
if run.Ended == nil {
|
||||
return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"}
|
||||
}
|
||||
if run.Outcome != meshwants.CompositionRunOutcomePassed {
|
||||
return machineEnd{couldNot: run.Said}
|
||||
}
|
||||
m, err := machineOf(run.Verdict)
|
||||
if err != nil {
|
||||
return machineEnd{couldNot: node + ": the run's verdict does not read: " + err.Error()}
|
||||
}
|
||||
if m.NotJudged != "" { // a step never passes on a machine it did not judge (A-467)
|
||||
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
|
||||
}
|
||||
why := strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")
|
||||
switch {
|
||||
case m.Blamed == nil && !m.Composes: // an old composer (§721)
|
||||
return machineEnd{judged: true, failed: node + " does not compose: " + why, warnings: []string{node + ": the composer names no culprits"}}
|
||||
case len(m.Blamed) == 0 && !m.Composes: // the node's own account (A-452)
|
||||
return machineEnd{judged: true, failed: node + " does not compose and blames no module: " + why}
|
||||
}
|
||||
end := machineEnd{judged: true}
|
||||
var own, others []string
|
||||
for _, b := range m.Blamed {
|
||||
switch {
|
||||
case b.Module == module && b.Problem == notTried:
|
||||
end.couldNot = fmt.Sprintf("%s holds %s untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", node, module)
|
||||
case b.Module == module:
|
||||
own = append(own, b.Problem)
|
||||
default:
|
||||
others = append(others, node+" holds "+b.Module+": "+b.Problem)
|
||||
}
|
||||
}
|
||||
if len(own) > 0 {
|
||||
end.failed, end.couldNot = node+" holds "+module+": "+strings.Join(own, "; "), ""
|
||||
}
|
||||
end.warnings = others
|
||||
return end
|
||||
}
|
||||
|
||||
// machineOf is the composer's verdict of one machine as a run keeps it.
|
||||
func machineOf(v map[string]any) (composer.Machine, error) {
|
||||
var m composer.Machine
|
||||
raw, err := json.Marshal(v)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &m)
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
// runVerdict reads the composer's answer of one machine: its verdict as the run keeps it, or why it is none.
|
||||
func runVerdict(node string, answer []byte) (map[string]any, string) {
|
||||
var v composer.Verdict
|
||||
if err := json.Unmarshal(answer, &v); err != nil {
|
||||
return machineEnd{couldNot: node + ": the composer answered no verdict: " + err.Error()}
|
||||
return nil, node + ": the composer answered no verdict: " + err.Error()
|
||||
}
|
||||
if v.V != composer.Version {
|
||||
return machineEnd{couldNot: fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)}
|
||||
return nil, fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)
|
||||
}
|
||||
for _, c := range v.Cases {
|
||||
if c.Name != changeCase {
|
||||
continue
|
||||
}
|
||||
m, ok := c.Machines[node]
|
||||
switch {
|
||||
case !ok:
|
||||
return machineEnd{couldNot: node + ": the composer's verdict names no such machine"}
|
||||
case m.NotJudged != "": // a step never passes on a machine it did not judge (A-467)
|
||||
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
|
||||
case !m.Composes:
|
||||
return machineEnd{judged: true, failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")}
|
||||
if !ok {
|
||||
return nil, node + ": the composer's verdict names no such machine"
|
||||
}
|
||||
return machineEnd{judged: true}
|
||||
raw, err := json.Marshal(m)
|
||||
var out map[string]any
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &out)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, node + ": the composer's verdict cannot be kept: " + err.Error()
|
||||
}
|
||||
return out, ""
|
||||
}
|
||||
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
|
||||
return nil, node + ": the composer's verdict has no case " + changeCase
|
||||
}
|
||||
|
||||
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
|
||||
@@ -400,20 +507,28 @@ func (r *compositionRun) seatAt(ctx context.Context, seat, digest string) (json.
|
||||
return def, nil
|
||||
}
|
||||
|
||||
// caseOf is the case's manifests and verdicts from the revision's entries: each manifest whole, and the verdict of each one not
|
||||
// accepted (a module without one is accepted).
|
||||
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, error) {
|
||||
// caseOf is the case's manifests, verdicts and none from the revision's entries: each manifest whole, the verdict of each one not
|
||||
// accepted (a module without one is accepted), and the revision's none words of each module it gives no fingerprint, whose
|
||||
// build can never be made (design 47 G A-470j, A-470k).
|
||||
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, map[string]string, error) {
|
||||
manifests := map[string]composer.Manifest{}
|
||||
verdicts := map[string]json.RawMessage{}
|
||||
var none map[string]string
|
||||
for name, e := range half.Entries {
|
||||
if len(e.Manifest) == 0 {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
|
||||
return nil, nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
|
||||
}
|
||||
manifests[name] = composer.Manifest{Raw: raw}
|
||||
if e.Fp == "" && e.None != "" {
|
||||
if none == nil {
|
||||
none = map[string]string{}
|
||||
}
|
||||
none[name] = e.None
|
||||
}
|
||||
if e.Verdict == "" || e.Verdict == mcc.VerdictAccepted {
|
||||
continue
|
||||
}
|
||||
@@ -426,11 +541,11 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes
|
||||
}
|
||||
raw, err = json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
verdicts[name] = raw
|
||||
}
|
||||
return manifests, verdicts, nil
|
||||
return manifests, verdicts, none, nil
|
||||
}
|
||||
|
||||
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
|
||||
@@ -468,50 +583,53 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
|
||||
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
|
||||
}
|
||||
|
||||
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
|
||||
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
|
||||
// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451).
|
||||
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
|
||||
// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243,
|
||||
// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the
|
||||
// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's
|
||||
// `running` is not heard gets the revision's entry, said (A-451).
|
||||
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, error), error) {
|
||||
engine, ok := nodeEngineEntry(half.Entries)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
|
||||
}
|
||||
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
|
||||
p, err := programOfEntry(engine)
|
||||
return func(string) (buildstep.RunProgram, error) { return p, err }, nil
|
||||
return func(string) (buildstep.RunProgram, string, error) { return p, engine.Fp, err }, nil
|
||||
}
|
||||
var mu sync.Mutex
|
||||
byFp := map[string]buildstep.RunProgram{}
|
||||
return func(node string) (buildstep.RunProgram, error) {
|
||||
return func(node string) (buildstep.RunProgram, string, error) {
|
||||
running, ok := r.running(node, engine.Module)
|
||||
if !ok || running.Fingerprint == "" {
|
||||
r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node,
|
||||
engine.Module, engine.Fp)
|
||||
return programOfEntry(engine)
|
||||
p, err := programOfEntry(engine)
|
||||
return p, engine.Fp, err
|
||||
}
|
||||
fp := running.Fingerprint
|
||||
mu.Lock()
|
||||
p, ok := byFp[running.Fingerprint]
|
||||
p, ok := byFp[fp]
|
||||
mu.Unlock()
|
||||
if ok {
|
||||
return p, nil
|
||||
return p, fp, nil
|
||||
}
|
||||
what := "the entry of the node-engine's build " + running.Fingerprint + " is not read"
|
||||
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
|
||||
what := "the entry of the node-engine's build " + fp + " is not read"
|
||||
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp})
|
||||
if err != nil || body == nil {
|
||||
return buildstep.RunProgram{}, unreadErr(what, err, said)
|
||||
return buildstep.RunProgram{}, fp, unreadErr(what, err, said)
|
||||
}
|
||||
var a meshcatalog.EntryAnswer
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err)
|
||||
return buildstep.RunProgram{}, fp, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
p, err = programOfEntry(a.Entry)
|
||||
if err != nil {
|
||||
return p, err
|
||||
return p, fp, err
|
||||
}
|
||||
mu.Lock()
|
||||
byFp[running.Fingerprint] = p
|
||||
byFp[fp] = p
|
||||
mu.Unlock()
|
||||
return p, nil
|
||||
return p, fp, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -30,12 +31,16 @@ type runStand struct {
|
||||
requests map[string]composer.Request // by node
|
||||
entries []string // the fingerprints `entry` was asked
|
||||
r *compositionRun
|
||||
runs *compositionRuns
|
||||
puts int // the run asks put
|
||||
// noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest)
|
||||
// pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read.
|
||||
noDefinitions bool
|
||||
trunkSeats []meshcatalog.SeatDefinitionsAnswerSeatsItem
|
||||
unjudged string
|
||||
listFault error
|
||||
// verdict, when set, is the composer's verdict of a machine (by node), else the stand's own.
|
||||
verdict func(node string) (composer.Machine, bool)
|
||||
}
|
||||
|
||||
func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand {
|
||||
@@ -52,7 +57,8 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
"": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}},
|
||||
Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}},
|
||||
Manifest: map[string]any{"module": "bad"}}}, Next: "bad"},
|
||||
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}}}},
|
||||
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}},
|
||||
{Module: "lidarr", None: "builders disagree: node", Manifest: map[string]any{"module": "lidarr"}}}}, // no fingerprint (A-470j)
|
||||
}
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
@@ -99,10 +105,10 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
away := &awayNodes{}
|
||||
away.set("shanks", true)
|
||||
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
ends := &runEnds{}
|
||||
s.r = &compositionRun{
|
||||
s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
|
||||
s.r = &compositionRun{runs: s.runs,
|
||||
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
|
||||
keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
|
||||
keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf,
|
||||
trunk: func() (mcc.Half, bool) {
|
||||
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
|
||||
},
|
||||
@@ -124,13 +130,27 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
s.asks[node], s.requests[node] = a, req
|
||||
s.mu.Unlock()
|
||||
m := composer.Machine{Composes: true}
|
||||
if node == s.unjudged {
|
||||
override, set := composer.Machine{}, false
|
||||
if s.verdict != nil {
|
||||
override, set = s.verdict(node)
|
||||
}
|
||||
if set {
|
||||
m = override
|
||||
} else if node == s.unjudged {
|
||||
m = composer.Machine{NotJudged: "the composer was handed no seat list"}
|
||||
} else if node == "g14" {
|
||||
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
|
||||
}
|
||||
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
|
||||
go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)})
|
||||
s.mu.Lock()
|
||||
s.puts++
|
||||
s.mu.Unlock()
|
||||
go func() {
|
||||
if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)},
|
||||
func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -146,7 +166,8 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
out := s.r.run(context.Background(), "ask-step", step)
|
||||
if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") ||
|
||||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "shanks is away") {
|
||||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 2 || !strings.Contains(out.Warnings[0], "shanks is away") ||
|
||||
out.Warnings[1] != "g14: the composer names no culprits" {
|
||||
t.Fatalf("the step came to %+v", out)
|
||||
}
|
||||
if len(s.asks) != 2 {
|
||||
@@ -160,12 +181,12 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
if s.asks["g14"].Run.Program.Commit != commitE2 {
|
||||
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
|
||||
}
|
||||
if req.Depth != composer.DepthCompose || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
|
||||
if req.Depth != composer.DepthCompose || req.BlameTries != 20 || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
|
||||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
|
||||
t.Errorf("the request of ace %+v", req)
|
||||
}
|
||||
c := req.Cases[0]
|
||||
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 3 || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
|
||||
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 4 || len(c.None) != 1 || c.None["lidarr"] != "builders disagree: node" || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
|
||||
!strings.Contains(string(c.Verdicts["bad"]), "no seat x") {
|
||||
t.Errorf("the case %+v %s", c.Manifests, c.Verdicts)
|
||||
}
|
||||
@@ -329,3 +350,227 @@ func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run
|
||||
// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen
|
||||
// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its
|
||||
// own runs, and a run that could not run is never read again.
|
||||
func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
|
||||
var wg sync.WaitGroup
|
||||
outs := make([]stepOutcome, 3)
|
||||
for i, m := range []string{"m", "m2", "m3"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
for i, o := range outs {
|
||||
if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" ||
|
||||
o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask
|
||||
t.Errorf("step %d names the runs %+v", i, o.Shared)
|
||||
}
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
|
||||
t.Errorf("step %d came to %+v, want both machines read", i, o)
|
||||
}
|
||||
}
|
||||
if s.puts != 2 {
|
||||
t.Fatalf("%d run asks put, want one per machine", s.puts)
|
||||
}
|
||||
ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace"))
|
||||
g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
|
||||
if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 ||
|
||||
g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) ||
|
||||
ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false {
|
||||
t.Errorf("the runs: ace %+v, g14 %+v", ace, g14)
|
||||
}
|
||||
if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` {
|
||||
t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req))
|
||||
}
|
||||
// Another candidate (a run on request) starts its own runs.
|
||||
if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 {
|
||||
t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts)
|
||||
}
|
||||
// A run whose ask could not be put could not run; the next step of the candidate starts it again.
|
||||
f := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
f.r.inputs.machines = func(string, string) []string { return []string{"ace"} }
|
||||
put, refused := f.r.put, 0
|
||||
f.r.put = func(ctx context.Context, a buildstep.StepAsk) error {
|
||||
if refused == 0 {
|
||||
refused++
|
||||
return errors.New("no stream")
|
||||
}
|
||||
return put(ctx, a)
|
||||
}
|
||||
first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
|
||||
if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 {
|
||||
t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// Design 47 G §721, §722, A-470b: a step judges each machine from its verdict's `blamed`: a verdict blaming the step's module fails
|
||||
// it and passes another module's step with a warning naming the module; a machine that does not compose and blames no module fails
|
||||
// every step; a verdict with no `blamed` is judged as before and says "the composer names no culprits"; a module the search did not
|
||||
// try is could not run for its own step; a run that could not run is could not run.
|
||||
func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) {
|
||||
at := time.Now()
|
||||
runOf := func(verdict string) meshwants.CompositionRun {
|
||||
var v map[string]any
|
||||
if err := json.Unmarshal([]byte(verdict), &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return meshwants.CompositionRun{Candidate: "main.7", Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomePassed, Verdict: v}
|
||||
}
|
||||
blamesA := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and c's"}]}`
|
||||
for _, tc := range []struct {
|
||||
name, module string
|
||||
run meshwants.CompositionRun
|
||||
failed, couldNo string
|
||||
warnings []string
|
||||
}{
|
||||
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil},
|
||||
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}},
|
||||
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil},
|
||||
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}},
|
||||
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil},
|
||||
{"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "",
|
||||
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil},
|
||||
{"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "",
|
||||
[]string{"ace holds a: not tried: the search reached its bound"}},
|
||||
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil},
|
||||
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
e := judgeRun(tc.module, tc.run)
|
||||
if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) {
|
||||
t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §727, A-468g, A-470d: a rerun of a composition step starts a new run only on the machine it did not pass, replacing
|
||||
// that key and carrying every former reader, the one that did not pass marked rerun with the rerun's caller; at the new run's end
|
||||
// that reader says its end again, marked rerun, naming the new run's ask; a reader that passed does not move; the machine the
|
||||
// step passed keeps its verdict.
|
||||
func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
|
||||
var mu sync.Mutex
|
||||
fixed := false
|
||||
s.verdict = func(node string) (composer.Machine, bool) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
m := composer.Machine{Composes: true}
|
||||
m.Blamed = []struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{}
|
||||
if node == "g14" && !fixed {
|
||||
m.Composes, m.Reason = false, "two modules own one path"
|
||||
m.Blamed = append(m.Blamed, struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{"a", "/etc/x"}, struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{"d", "/etc/x"})
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
ends := make(chan meshwants.StepEnded, 16)
|
||||
msgIDs := map[string]string{} // the message id of each step's latest end
|
||||
steps := &compositionSteps{node: "n1", now: time.Now, logf: t.Logf, run: s.r.run, resume: s.r.resume, ended: s.runs.stepEnded,
|
||||
say: func(event string, body any, msgID string) error {
|
||||
if e, ok := body.(meshwants.StepEnded); ok {
|
||||
mu.Lock()
|
||||
msgIDs[e.ID] = msgID
|
||||
mu.Unlock()
|
||||
ends <- e
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
move := steps.move(context.Background())
|
||||
s.runs.moved.Store(&move)
|
||||
step := func(m string, rerun bool) []byte {
|
||||
raw, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition, Rerun: rerun, By: map[bool]string{true: "op@n1"}[rerun]})
|
||||
return raw
|
||||
}
|
||||
got := map[string]meshwants.StepEnded{}
|
||||
for _, m := range []string{"a", "b", "d"} {
|
||||
steps.take(context.Background(), "ask-"+m, step(m, false))
|
||||
e := <-ends
|
||||
got[e.Module] = e
|
||||
}
|
||||
if got["a"].Outcome != meshwants.StepEndedOutcomeFailed || got["d"].Outcome != meshwants.StepEndedOutcomeFailed || got["b"].Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Fatalf("the first ends: %+v", got)
|
||||
}
|
||||
former := got["d"].Shared[1].Ask
|
||||
mu.Lock()
|
||||
fixed = true
|
||||
mu.Unlock()
|
||||
steps.take(context.Background(), "ask-a2", step("a", true))
|
||||
moved := map[string]meshwants.StepEnded{}
|
||||
for range 2 {
|
||||
select {
|
||||
case e := <-ends:
|
||||
moved[e.ID] = e
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the ends after the rerun: %+v", moved)
|
||||
}
|
||||
}
|
||||
a, d := moved["ask-a2"], moved["ask-d"]
|
||||
if a.Outcome != meshwants.StepEndedOutcomePassed || d.Outcome != meshwants.StepEndedOutcomePassed || !d.Rerun || d.By != "op@n1" ||
|
||||
len(d.Shared) != 2 || d.Shared[1].Machine != "g14" || d.Shared[1].Ask == former || d.Shared[1].Ask != a.Shared[1].Ask {
|
||||
t.Fatalf("after the rerun: a %+v, d %+v", a, d)
|
||||
}
|
||||
mu.Lock()
|
||||
// A-470q: the first end keeps the ask's id; the end said again names the moved run's ask, so the bus does not drop it.
|
||||
if msgIDs["ask-a2"] != "ask-a2" || msgIDs["ask-d"] != "ask-d."+a.Shared[1].Ask || msgIDs["ask-b"] != "ask-b" {
|
||||
t.Errorf("the message ids of the ends: %v", msgIDs)
|
||||
}
|
||||
mu.Unlock()
|
||||
if s.puts != 3 {
|
||||
t.Errorf("%d run asks, want one more for the machine a did not pass", s.puts)
|
||||
}
|
||||
select {
|
||||
case e := <-ends:
|
||||
t.Errorf("a reader that passed moved: %+v", e)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
run, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
|
||||
marks := map[string]bool{}
|
||||
for _, c := range run.Readers {
|
||||
marks[c.ID] = c.Rerun
|
||||
}
|
||||
if len(run.Readers) != 3 || !marks["ask-d"] || marks["ask-b"] || !marks["ask-a2"] {
|
||||
t.Errorf("the new run's readers %+v", run.Readers)
|
||||
}
|
||||
// A-470m: each reader's `ended` is written once its step said its end, and ask-b's carried into the new run.
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
wake := s.runs.wake()
|
||||
run, _, _, _ = s.runs.read(runKeyOf("main.7", "g14"))
|
||||
if !slices.ContainsFunc(run.Readers, func(c meshwants.CompositionReader) bool { return c.Ended == nil }) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-wake:
|
||||
case <-deadline:
|
||||
t.Fatalf("a reader of the new run names no end: %+v", run.Readers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,896 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The composition runs (design 47 G §718, §719; A-468, A-469d, A-469w, A-469x): the assigner composes each machine once per
|
||||
// candidate, whatever number of composition steps name it. Each run is one key of its old state `composition-runs`,
|
||||
// `<candidate key>.<machine>`, each part escaped as a key's name; the step that starts a run writes the key create-only,
|
||||
// so two steps of one candidate starting at once start one run and the second reads it. A reader is the composition step's
|
||||
// StepRun with its ask's id (A-468), added and removed by compare-and-set. A key is deleted once composition-runs-kept has
|
||||
// passed since its `ended`, judged at each new run over at most 20 of the oldest ended keys, as the state inputs is
|
||||
// (A-465b, A-468b); at the cap (composition-runs-cap-mib, the manifest's max-mib its ceiling, A-469d) no run is started.
|
||||
|
||||
const (
|
||||
// runsSwept is how many of the oldest ended keys each new run judges.
|
||||
runsSwept = 20
|
||||
// runBoundPrefix and runUntakenPrefix open a run's two bound keys, each with its own time to live (§720, A-468b,
|
||||
// A-470): `bound.<run key>` from the ask's ask-taken, `untaken.<run key>` from the run's start.
|
||||
runBoundPrefix = "bound."
|
||||
runUntakenPrefix = "untaken."
|
||||
)
|
||||
|
||||
// runKeyOf is the key of a candidate's composition run of a machine (§719).
|
||||
func runKeyOf(candidate, machine string) string {
|
||||
return keysPart(candidate) + "." + keys.EscapeName(machine)
|
||||
}
|
||||
|
||||
// keysPart is a name as a key's part.
|
||||
func keysPart(name string) string { return keys.EscapeName(name) }
|
||||
|
||||
// isRunKey is whether a key of the state is a run's own, not one of its bound keys.
|
||||
func isRunKey(key string) bool {
|
||||
return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix)
|
||||
}
|
||||
|
||||
// runSettings are the settings of the composition runs, each read at the start from the file the manifest declares.
|
||||
type runSettings struct {
|
||||
// capBytes is composition-runs-cap-mib in bytes.
|
||||
capBytes int64
|
||||
// kept is composition-runs-kept, stepBound composition-step-bound, untaken run-untaken-bound (A-469x).
|
||||
kept, stepBound, untaken time.Duration
|
||||
// blameTries is blame-tries, carried in each run's request (§721).
|
||||
blameTries int
|
||||
}
|
||||
|
||||
// defaultRunSettings are the settings' defaults, as the manifest gives them.
|
||||
var defaultRunSettings = runSettings{capBytes: 16 << 20, kept: 24 * time.Hour, stepBound: 30 * time.Minute, untaken: 24 * time.Hour, blameTries: 20}
|
||||
|
||||
// runSettingsOf reads the settings composition-runs-cap-mib, composition-runs-kept, composition-step-bound,
|
||||
// run-untaken-bound and blame-tries from the files the environment names (A-469x, as outsideSettings): a missing or empty
|
||||
// file is its default; a content that is no positive whole number or Go duration is an error naming the file.
|
||||
func runSettingsOf(getenv func(string) string) (runSettings, error) {
|
||||
s := defaultRunSettings
|
||||
mib, err := settingFile(getenv("MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE"))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if mib > 0 {
|
||||
s.capBytes = int64(mib) << 20
|
||||
}
|
||||
tries, err := settingFile(getenv("MESH_ASSIGNER_BLAME_TRIES_FILE"))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if tries > 0 {
|
||||
s.blameTries = tries
|
||||
}
|
||||
for env, to := range map[string]*time.Duration{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": &s.kept,
|
||||
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": &s.stepBound, "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": &s.untaken} {
|
||||
d, err := durationFile(getenv(env))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if d > 0 {
|
||||
*to = d
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// durationFile reads one setting's file holding a Go duration: 0 for no path, no file or an empty one.
|
||||
func durationFile(path string) (time.Duration, error) {
|
||||
if path == "" {
|
||||
return 0, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return 0, nil
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive Go duration such as 30m", path, v)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// errRunChanged is a compare-and-set of a run's key that met another write.
|
||||
var errRunChanged = errors.New("the composition run's key changed")
|
||||
|
||||
// runsKV is the state composition-runs: the bus bucket, or a test's.
|
||||
type runsKV interface {
|
||||
// get is a key's value and revision; nil for a key not there.
|
||||
get(key string) ([]byte, uint64, error)
|
||||
// create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so.
|
||||
create(key string, value []byte) (uint64, error)
|
||||
update(key string, value []byte, revision uint64) (uint64, error)
|
||||
// delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write
|
||||
// of the state is made from a value read before its latest change.
|
||||
delAt(key string, revision uint64) error
|
||||
keys() ([]string, error)
|
||||
// arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's
|
||||
// latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It
|
||||
// answers the revision written.
|
||||
arm(key, ask string, ttl time.Duration) (uint64, error)
|
||||
// used is how many bytes the state holds.
|
||||
used() (int64, error)
|
||||
}
|
||||
|
||||
// busRuns is runsKV over the bus bucket.
|
||||
type busRuns struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (b busRuns) get(key string) ([]byte, uint64, error) {
|
||||
e, err := b.kv.Get(b.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value(), e.Revision(), nil
|
||||
}
|
||||
|
||||
func (b busRuns) create(key string, value []byte) (uint64, error) {
|
||||
rev, err := b.kv.Create(b.ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (b busRuns) update(key string, value []byte, revision uint64) (uint64, error) {
|
||||
rev, err := b.kv.Update(b.ctx, key, value, revision)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (b busRuns) delAt(key string, revision uint64) error {
|
||||
err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return errRunChanged
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busRuns) keys() ([]string, error) {
|
||||
l, err := b.kv.ListKeys(b.ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for k := range l.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
|
||||
type boundValue struct {
|
||||
Bound string `json:"bound"`
|
||||
Ask string `json:"ask"`
|
||||
}
|
||||
|
||||
// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete
|
||||
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
|
||||
// takes its time to live with it, so the expiry is heard only ttl after the last write.
|
||||
func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
if b.js == nil {
|
||||
return 0, errors.New("no bus to arm a bound on")
|
||||
}
|
||||
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
subject := "$KV." + b.kv.Bucket() + "." + key
|
||||
for {
|
||||
st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var last uint64
|
||||
m, err := st.GetLastMsgForSubject(b.ctx, subject)
|
||||
switch {
|
||||
case err == nil:
|
||||
last = m.Sequence
|
||||
case !errors.Is(err, jetstream.ErrMsgNotFound):
|
||||
return 0, err
|
||||
}
|
||||
ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
|
||||
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return ack.Sequence, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b busRuns) used() (int64, error) {
|
||||
st, err := b.kv.Status(b.ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(st.Bytes()), nil
|
||||
}
|
||||
|
||||
// compositionRuns keeps the composition runs in the state composition-runs.
|
||||
type compositionRuns struct {
|
||||
kv runsKV
|
||||
set runSettings
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
// changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again.
|
||||
changed chan struct{}
|
||||
// seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is
|
||||
// being written at that moment, so every lookup takes these beside it (keys).
|
||||
seen map[string]bool
|
||||
// say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119);
|
||||
// nil says or drops nothing.
|
||||
say publisher
|
||||
drop func(ask, why string) error
|
||||
// moved is told each run that ended, for the readers a rerun carried to say their end again (compositionSteps.move).
|
||||
// Set once the steps are served, after the hub starts.
|
||||
moved atomic.Pointer[func(run meshwants.CompositionRun)]
|
||||
}
|
||||
|
||||
func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns {
|
||||
return &compositionRuns{kv: kv, set: set, now: now, logf: logf, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
// heard takes a change of the state from its watch (a module watches its own state, principle 1).
|
||||
func (r *compositionRuns) heard() {
|
||||
r.mu.Lock()
|
||||
close(r.changed)
|
||||
r.changed = make(chan struct{})
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// saw keeps a key the state's watch handed: standing, or deleted or expired.
|
||||
func (r *compositionRuns) saw(key string, standing bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.seen == nil {
|
||||
r.seen = map[string]bool{}
|
||||
}
|
||||
if standing {
|
||||
r.seen[key] = true
|
||||
} else {
|
||||
delete(r.seen, key)
|
||||
}
|
||||
}
|
||||
|
||||
// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later
|
||||
// is skipped by its reader.
|
||||
func (r *compositionRuns) keys() ([]string, error) {
|
||||
all, err := r.kv.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
for key := range r.seen {
|
||||
all = append(all, key)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
slices.Sort(all)
|
||||
return slices.Compact(all), nil
|
||||
}
|
||||
|
||||
// wake is closed at the next change heard.
|
||||
func (r *compositionRuns) wake() <-chan struct{} {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.changed
|
||||
}
|
||||
|
||||
// read is a run's value and revision, or false.
|
||||
func (r *compositionRuns) read(key string) (meshwants.CompositionRun, uint64, bool, error) {
|
||||
raw, rev, err := r.kv.get(key)
|
||||
if err != nil || raw == nil {
|
||||
return meshwants.CompositionRun{}, 0, false, err
|
||||
}
|
||||
var run meshwants.CompositionRun
|
||||
if err := json.Unmarshal(raw, &run); err != nil {
|
||||
return meshwants.CompositionRun{}, 0, false, fmt.Errorf("the composition run %s does not read: %w", key, err)
|
||||
}
|
||||
return run, rev, true, nil
|
||||
}
|
||||
|
||||
// change writes a run's key by compare-and-set: f changes the value read and says whether to write it; false leaves it.
|
||||
// It answers the value written or left, and false for a key not there.
|
||||
func (r *compositionRuns) change(key string, f func(*meshwants.CompositionRun) bool) (meshwants.CompositionRun, bool, error) {
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil || !ok {
|
||||
return run, ok, err
|
||||
}
|
||||
if !f(&run) {
|
||||
return run, true, nil
|
||||
}
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return run, true, err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return run, true, err
|
||||
}
|
||||
r.heard()
|
||||
return run, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// fill is the state's fill against its cap.
|
||||
func (r *compositionRuns) fill() (meshwants.Fill, error) {
|
||||
at := r.now().UTC()
|
||||
used, err := r.kv.used()
|
||||
return meshwants.Fill{Store: buckets.CompositionRunsState, Bytes: used, Cap: r.set.capBytes, At: &at}, err
|
||||
}
|
||||
|
||||
// joined is a reader's place in a run: the run's key, whether this reader started it (so it composes the machine), and the
|
||||
// value as it stood.
|
||||
type joined struct {
|
||||
key string
|
||||
started bool
|
||||
run meshwants.CompositionRun
|
||||
}
|
||||
|
||||
// join finds or starts the candidate's run of the machine for reader (§718, §719, §720): a run that stands and has not
|
||||
// ended, or that ended other than could not run, is read, the reader added; none, or one that could not run, is started:
|
||||
// written create-only (or replacing the ended key by compare-and-set), the reader its first, ask its run ask's id. At the
|
||||
// cap no run is started, an error naming the fill.
|
||||
func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
|
||||
key := runKeyOf(candidate, machine)
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if ok && (run.Ended == nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) {
|
||||
if hasReader(run.Readers, reader.ID) {
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
run.Readers = append(run.Readers, reader)
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
r.heard()
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) {
|
||||
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
|
||||
}
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
if err := r.room(); err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
started := r.now().UTC()
|
||||
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started,
|
||||
Readers: []meshwants.CompositionReader{reader}}
|
||||
raw, err := json.Marshal(fresh)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if ok {
|
||||
_, err = r.kv.update(key, raw, rev) // could not run is never read: a new run replaces it (§720)
|
||||
} else {
|
||||
_, err = r.kv.create(key, raw)
|
||||
}
|
||||
if errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
r.heard()
|
||||
// An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n).
|
||||
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
|
||||
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
|
||||
}
|
||||
r.sayWaits(fresh, fresh.Readers, "")
|
||||
return joined{key: key, started: true, run: fresh}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// room sweeps the state and says whether a new run may start: an error naming the fill at the cap (as A-465a).
|
||||
func (r *compositionRuns) room() error {
|
||||
if err := r.sweep(); err != nil {
|
||||
r.logf("composition-runs: the keys past composition-runs-kept are not all deleted: %v", err)
|
||||
}
|
||||
f, err := r.fill()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the fill of the state %s is not read, so no composition run is started: %w", buckets.CompositionRunsState, err)
|
||||
}
|
||||
if f.Bytes >= f.Cap {
|
||||
return fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so no composition run is started", buckets.CompositionRunsState,
|
||||
f.Cap>>20, f.Bytes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys.
|
||||
func (r *compositionRuns) sweep() error {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type endedAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var ended []endedAt
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || run.Ended == nil {
|
||||
continue
|
||||
}
|
||||
ended = append(ended, endedAt{key, *run.Ended})
|
||||
}
|
||||
slices.SortFunc(ended, func(a, b endedAt) int { return a.at.Compare(b.at) })
|
||||
if len(ended) > runsSwept {
|
||||
ended = ended[:runsSwept]
|
||||
}
|
||||
var errs []error
|
||||
for _, e := range ended {
|
||||
if r.now().Sub(e.at) >= r.set.kept {
|
||||
errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool {
|
||||
return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept
|
||||
}))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the
|
||||
// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand.
|
||||
func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error {
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil || !ok || !gone(run) {
|
||||
return err
|
||||
}
|
||||
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
r.heard()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at
|
||||
// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted.
|
||||
func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error {
|
||||
var errs []error
|
||||
for _, prefix := range prefixes {
|
||||
for {
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
break
|
||||
}
|
||||
if ok && run.Ask != ask {
|
||||
break // a newer run holds the key: its bounds are its own
|
||||
}
|
||||
held, rev, standing, err := r.boundOf(prefix + key)
|
||||
if err != nil || !standing {
|
||||
errs = append(errs, err)
|
||||
break
|
||||
}
|
||||
if held != ask {
|
||||
break // the bound of another ask (A-470n): not this run's to delete
|
||||
}
|
||||
if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// stepEnded writes at as the `ended` of the reader asked under id in each run of its step's candidate naming it, by
|
||||
// compare-and-set (A-470m): the time its step said its end, carried with it into a rerun's run.
|
||||
func (r *compositionRuns) stepEnded(id string, step meshwants.StepRun, at time.Time) {
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
|
||||
return
|
||||
}
|
||||
keys, err := r.keysOf(candidate, id)
|
||||
if err != nil {
|
||||
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
|
||||
return
|
||||
}
|
||||
at = at.UTC()
|
||||
for _, key := range keys {
|
||||
if _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
i := slices.IndexFunc(c.Readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
c.Readers[i].Ended = &at
|
||||
return true
|
||||
}); err != nil {
|
||||
r.logf("the end of the composition step %s is not written on the composition run %s: %v", id, key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hasReader is whether readers holds the step asked under id.
|
||||
func hasReader(readers []meshwants.CompositionReader, id string) bool {
|
||||
return slices.ContainsFunc(readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
|
||||
}
|
||||
|
||||
// readerOf is a composition step as a run's reader: its StepRun with its ask's id (A-468).
|
||||
func readerOf(id string, s meshwants.StepRun) meshwants.CompositionReader {
|
||||
return meshwants.CompositionReader{ID: id, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Branch: s.Branch, Commit: s.Commit,
|
||||
Revision: s.Revision, Source: s.Source, Fingerprint: s.Fingerprint, Run: s.Run, Rerun: s.Rerun, By: s.By}
|
||||
}
|
||||
|
||||
// stepRunOf is a reader's StepRun again, for its step-ended.
|
||||
func stepRunOf(c meshwants.CompositionReader) meshwants.StepRun {
|
||||
return meshwants.StepRun{Pipeline: c.Pipeline, Step: c.Step, Module: c.Module, Branch: c.Branch, Commit: c.Commit, Revision: c.Revision,
|
||||
Source: c.Source, Fingerprint: c.Fingerprint, Run: c.Run, Rerun: c.Rerun, By: c.By}
|
||||
}
|
||||
|
||||
// watchRuns hears the changes of the state composition-runs (principle 1): each change wakes the steps waiting on a run, and
|
||||
// the expiry of a bound key, the bus's marker, is handed to expired (§720). After the values that stand, ready is called.
|
||||
func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, expired func(key string), ready func()) error {
|
||||
w, err := kv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = w.Stop() }()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
if ready != nil {
|
||||
ready()
|
||||
ready = nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut)
|
||||
if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil {
|
||||
expired(e.Key())
|
||||
}
|
||||
r.heard()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers
|
||||
// false when the key holds another run or the run has ended already.
|
||||
func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
|
||||
ended := false
|
||||
_, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
at := r.now().UTC()
|
||||
c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict
|
||||
ended = true
|
||||
return true
|
||||
})
|
||||
return ended, err
|
||||
}
|
||||
|
||||
// keyOfAsk is the key of the run whose ask is ask, or none.
|
||||
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return "", meshwants.CompositionRun{}, err
|
||||
}
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err == nil && ok && run.Ask == ask {
|
||||
return key, run, nil
|
||||
}
|
||||
}
|
||||
return "", meshwants.CompositionRun{}, nil
|
||||
}
|
||||
|
||||
// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed
|
||||
// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run
|
||||
// names is another asker's, or of a run gone, and changes nothing.
|
||||
func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error {
|
||||
if e.Asker != runAsker || e.Ask == "" {
|
||||
return nil
|
||||
}
|
||||
key, run, err := r.keyOfAsk(e.Ask)
|
||||
if err != nil || key == "" {
|
||||
return err
|
||||
}
|
||||
if run.Ended != nil { // ended by its bound or a drop: the end heard later is ignored (§720)
|
||||
r.logf("the step-ended of the run ask %s is ignored: its composition run %s ended before it (%s)", e.Ask, key, run.Said)
|
||||
return nil
|
||||
}
|
||||
node := run.Machine
|
||||
outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, ""
|
||||
var verdict map[string]any
|
||||
if e.Outcome != buildstep.OutcomePassed {
|
||||
said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said)
|
||||
} else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254)
|
||||
said = node + ": " + err.Error()
|
||||
} else if verdict, said = runVerdict(node, answer); said == "" {
|
||||
outcome = meshwants.CompositionRunOutcomePassed
|
||||
}
|
||||
_, err = r.endRun(key, e.Ask, outcome, said, verdict)
|
||||
return err
|
||||
}
|
||||
|
||||
// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it
|
||||
// stands then; a key gone could not run.
|
||||
func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun {
|
||||
out := map[string]meshwants.CompositionRun{}
|
||||
for {
|
||||
wake := r.wake()
|
||||
open := false
|
||||
for _, key := range keys {
|
||||
run, _, ok, err := r.read(key)
|
||||
switch {
|
||||
case err != nil:
|
||||
open = true
|
||||
continue
|
||||
case !ok:
|
||||
at := r.now().UTC()
|
||||
candidate, machine, _ := strings.Cut(key, ".")
|
||||
run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at,
|
||||
Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"}
|
||||
case run.Ended == nil:
|
||||
open = true
|
||||
}
|
||||
out[key] = run
|
||||
}
|
||||
if !open {
|
||||
return out
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return out
|
||||
case <-wake:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unescape is a key's part as a name again; a part that does not read stays as it is.
|
||||
func unescape(part string) string {
|
||||
if name, err := keys.UnescapeName(part); err == nil {
|
||||
return name
|
||||
}
|
||||
return part
|
||||
}
|
||||
|
||||
// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands.
|
||||
func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared {
|
||||
out := []meshwants.Shared{}
|
||||
for _, key := range keys {
|
||||
run, ok := runs[key]
|
||||
if !ok {
|
||||
if run, _, ok, _ = r.read(key); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers
|
||||
// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
|
||||
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
|
||||
func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
var errs []error
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
var left []string
|
||||
var leaving []meshwants.CompositionReader
|
||||
run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
left, leaving = nil, nil
|
||||
keep := c.Readers[:0:0]
|
||||
for _, reader := range c.Readers {
|
||||
if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition {
|
||||
left = append(left, reader.ID)
|
||||
leaving = append(leaving, reader)
|
||||
continue
|
||||
}
|
||||
keep = append(keep, reader)
|
||||
}
|
||||
c.Readers = keep
|
||||
return len(left) > 0
|
||||
})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if !ok || len(left) == 0 {
|
||||
continue
|
||||
}
|
||||
ids = append(ids, left...)
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) { // a reader leaving a run not yet taken: its wait ends at once (A-470u)
|
||||
r.sayWaits(run, leaving, "superseded")
|
||||
}
|
||||
if stop != nil {
|
||||
stop(left) // the steps that left say no end, the run's included
|
||||
}
|
||||
if run.Ended != nil || len(run.Readers) > 0 {
|
||||
continue
|
||||
}
|
||||
why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled"
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, why); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return ids, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there.
|
||||
func passedOn(module string, run meshwants.CompositionRun) bool {
|
||||
e := judgeRun(module, run)
|
||||
return e.failed == "" && e.couldNot == ""
|
||||
}
|
||||
|
||||
// formerOf is the place in readers of a step's former ask: the reader of the same pipeline's composition step, or -1.
|
||||
func formerOf(readers []meshwants.CompositionReader, pipeline string) int {
|
||||
return slices.IndexFunc(readers, func(c meshwants.CompositionReader) bool {
|
||||
return c.Pipeline == pipeline && c.Step == meshwants.CompositionReaderStepComposition
|
||||
})
|
||||
}
|
||||
|
||||
// rejoin is join for a step asked again (§727, A-468g, A-470d): on a machine whose ended run it passed it keeps that verdict,
|
||||
// reading the run under its new ask; a running run it joins; an ended run it did not read and that could run it reads, as one
|
||||
// started after its last end; otherwise it starts a new run, replacing the ended key by compare-and-set and carrying every former
|
||||
// reader into the new value: each that did not pass on the former run marked `rerun` with by the rerun's caller, each that passed
|
||||
// not (A-470d).
|
||||
func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
|
||||
key := runKeyOf(candidate, machine)
|
||||
for {
|
||||
run, rev, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if !ok {
|
||||
return r.join(reader, candidate, machine, ask)
|
||||
}
|
||||
at := formerOf(run.Readers, reader.Pipeline)
|
||||
ended := run.Ended != nil
|
||||
keep := ended && at >= 0 && passedOn(reader.Module, run)
|
||||
read := !ended || (at < 0 && run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun)
|
||||
if keep || read {
|
||||
if at >= 0 && run.Readers[at].ID == reader.ID {
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
var former []meshwants.CompositionReader
|
||||
if at >= 0 {
|
||||
former = []meshwants.CompositionReader{run.Readers[at]}
|
||||
run.Readers[at] = reader
|
||||
} else {
|
||||
run.Readers = append(run.Readers, reader)
|
||||
}
|
||||
if err := r.write(key, run, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if run.Ended == nil && r.untaken(key, run.Ask) { // as join: a reader of a run not yet taken waits for a build node (A-469y)
|
||||
r.sayWaits(run, former, "superseded") // the place it takes: that wait ends at once (A-470u)
|
||||
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
|
||||
}
|
||||
return joined{key: key, run: run}, nil
|
||||
}
|
||||
if err := r.room(); err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
started := r.now().UTC()
|
||||
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started}
|
||||
for i, c := range run.Readers {
|
||||
if i == at {
|
||||
continue
|
||||
}
|
||||
c.Rerun, c.By = !passedOn(c.Module, run), ""
|
||||
if c.Rerun {
|
||||
c.By = reader.By
|
||||
}
|
||||
fresh.Readers = append(fresh.Readers, c)
|
||||
}
|
||||
fresh.Readers = append(fresh.Readers, reader)
|
||||
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it
|
||||
r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err)
|
||||
}
|
||||
if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return joined{}, err
|
||||
}
|
||||
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
|
||||
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
|
||||
}
|
||||
r.sayWaits(fresh, fresh.Readers, "")
|
||||
return joined{key: key, started: true, run: fresh}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// write writes a run's value at its revision.
|
||||
func (r *compositionRuns) write(key string, run meshwants.CompositionRun, rev uint64) error {
|
||||
raw, err := json.Marshal(run)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.kv.update(key, raw, rev); err != nil {
|
||||
return err
|
||||
}
|
||||
r.heard()
|
||||
return nil
|
||||
}
|
||||
+404
@@ -0,0 +1,404 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes
|
||||
// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and
|
||||
// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a
|
||||
// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails
|
||||
// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound,
|
||||
// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never
|
||||
// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not
|
||||
// pass and moves b; a superseded step leaves
|
||||
// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the
|
||||
// readers and deletes the key; at the cap a step ends could not run, naming the fill.
|
||||
func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart
|
||||
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"}
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
const commit = "1111111111111111111111111111111111111111"
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
|
||||
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
|
||||
nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject},
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
awaitKept(t, b, nodeengine.StreamRunning, 2)
|
||||
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".",
|
||||
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
|
||||
b.cat.mu.Lock()
|
||||
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine}
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}}
|
||||
}
|
||||
b.cat.rev[trunkBranch] = 3
|
||||
b.cat.mu.Unlock()
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
b.place(t, node, m)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
inputsAsked := map[string]int{} // by candidate and node
|
||||
var drops []string
|
||||
respond := func(subject string, f func(m *nats.Msg) any) {
|
||||
sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"})
|
||||
_ = m.Respond(raw)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any {
|
||||
node := strings.Split(m.Subject, ".")[5]
|
||||
var in nodeengine.ComposeCandidateInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
mu.Lock()
|
||||
inputsAsked[in.Candidate+" "+node]++
|
||||
mu.Unlock()
|
||||
return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}
|
||||
})
|
||||
respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} })
|
||||
respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any {
|
||||
var in nodebuildgo.DropInput
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
mu.Lock()
|
||||
drops = append(drops, in.ID)
|
||||
mu.Unlock()
|
||||
return map[string]any{}
|
||||
})
|
||||
respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} })
|
||||
// The ends are read from the stream through a durable consumer, as the runner hears them: an end the bus drops as a
|
||||
// repeated message id is never heard (A-470q).
|
||||
ended := make(chan meshwants.StepEnded, 32)
|
||||
runner, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "runner-stand-in",
|
||||
FilterSubject: "mesh.seat.mesh-wants.event.step-ended.n1", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard, err := runner.Consume(func(m jetstream.Msg) {
|
||||
var e meshwants.StepEnded
|
||||
_ = json.Unmarshal(m.Data(), &e)
|
||||
_ = m.Ack()
|
||||
ended <- e
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(heard.Stop)
|
||||
subscribed(t, nc)
|
||||
|
||||
from := b.journal.mark()
|
||||
_, stop := b.startStoppable(t)
|
||||
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
|
||||
select {
|
||||
case <-b.seatListRead:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the assigner did not read the seat list within 30 s")
|
||||
}
|
||||
queue, _ := buildstep.AskStream(buildstep.AskGo)
|
||||
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants",
|
||||
AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
step := func(id, module, run string, rerun bool) {
|
||||
t.Helper()
|
||||
s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3,
|
||||
Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun}
|
||||
if run != "" {
|
||||
s.Pipeline = "run-" + run + "." + module
|
||||
}
|
||||
if rerun {
|
||||
s.By = "op@n1"
|
||||
}
|
||||
raw, _ := json.Marshal(s)
|
||||
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Data = raw
|
||||
if err := nc.PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
type asked struct {
|
||||
ask buildstep.StepAsk
|
||||
machine string
|
||||
}
|
||||
// asks takes n run asks of the stand-in build seat, each with the machine its request composes.
|
||||
asks := func(n int) []asked {
|
||||
t.Helper()
|
||||
var out []asked
|
||||
for range n {
|
||||
got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("%d of %d run asks put: %v", len(out), n, err)
|
||||
}
|
||||
_ = got.Ack()
|
||||
a, err := buildstep.DecodeStepAsk(got.Data())
|
||||
if err != nil || a.Run == nil {
|
||||
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
|
||||
}
|
||||
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
|
||||
if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 {
|
||||
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
|
||||
}
|
||||
out = append(out, asked{a, req.Machines[0]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
noMoreAsks := func(what string) {
|
||||
t.Helper()
|
||||
if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil {
|
||||
t.Fatalf("%s: one more run ask is put: %s", what, got.Data())
|
||||
}
|
||||
}
|
||||
say := func(event string, node string, body any) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(body)
|
||||
if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
answer := func(a asked, machine string) {
|
||||
t.Helper()
|
||||
v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine)
|
||||
say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()})
|
||||
}
|
||||
ends := func(n int, what string) map[string]meshwants.StepEnded {
|
||||
t.Helper()
|
||||
out := map[string]meshwants.StepEnded{}
|
||||
for len(out) < n {
|
||||
select {
|
||||
case e := <-ended:
|
||||
out[e.ID] = e
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
runKeys := func(candidate string) []string {
|
||||
all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx)
|
||||
var out []string
|
||||
for _, k := range all {
|
||||
if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// readers waits, through a watch of the key, until the run of key has n readers.
|
||||
readers := func(key string, n int) {
|
||||
t.Helper()
|
||||
w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
var run meshwants.CompositionRun
|
||||
if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n {
|
||||
return
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the run %s has not %d readers", key, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}`
|
||||
composes := `{"composes":true,"blamed":[]}`
|
||||
|
||||
// One candidate, three steps, two machines: two runs.
|
||||
for _, m := range []string{"a", "b", "c"} {
|
||||
step("step-"+m, m, "", false)
|
||||
}
|
||||
first := asks(2)
|
||||
noMoreAsks("three steps of one candidate")
|
||||
for _, a := range first {
|
||||
answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine])
|
||||
}
|
||||
got := ends(3, "the three steps")
|
||||
if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") ||
|
||||
bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 ||
|
||||
!strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask {
|
||||
t.Fatalf("the steps of one candidate: %+v", got)
|
||||
}
|
||||
if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 {
|
||||
t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked)
|
||||
}
|
||||
|
||||
// Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail.
|
||||
step("step-r1", "a", "r1", false)
|
||||
for _, a := range asks(2) {
|
||||
answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine])
|
||||
}
|
||||
if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") ||
|
||||
len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" {
|
||||
t.Fatalf("a step of another candidate: %+v", e)
|
||||
}
|
||||
|
||||
// The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken.
|
||||
from = b.journal.mark()
|
||||
step("step-r2", "a", "r2", false)
|
||||
bound := asks(2)
|
||||
var n1 asked
|
||||
for _, a := range bound {
|
||||
if a.machine == "n1" {
|
||||
n1 = a
|
||||
}
|
||||
}
|
||||
say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"})
|
||||
for i := range 3 { // past the 2 s bound, each beat within it
|
||||
select {
|
||||
case e := <-ended:
|
||||
t.Fatalf("a run saying its progress ended: %+v", e)
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n)
|
||||
runs := b.buckets[buckets.CompositionRuns()]
|
||||
all, _ := runs.Keys(b.ctx)
|
||||
held := ""
|
||||
for _, k := range all {
|
||||
if strings.HasPrefix(k, runBoundPrefix) {
|
||||
if e, err := runs.Get(b.ctx, k); err == nil {
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(e.Value(), &v)
|
||||
held += v.Ask
|
||||
}
|
||||
}
|
||||
}
|
||||
if held != n1.ask.ID {
|
||||
t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID)
|
||||
}
|
||||
}
|
||||
say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"})
|
||||
}
|
||||
e := ends(1, "the bounds")["step-r2"]
|
||||
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") ||
|
||||
!strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") {
|
||||
t.Fatalf("past the bounds: %+v", e)
|
||||
}
|
||||
mu.Lock()
|
||||
dropped := len(drops)
|
||||
mu.Unlock()
|
||||
if dropped != 2 {
|
||||
t.Fatalf("%d asks dropped past their bounds, want 2", dropped)
|
||||
}
|
||||
answer(n1, composes) // heard after the bound: ignored and journaled
|
||||
b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second)
|
||||
|
||||
// A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves.
|
||||
step("step-a2", "a", "", true)
|
||||
again := asks(1)
|
||||
noMoreAsks("a rerun of a")
|
||||
if again[0].machine != "n2" {
|
||||
t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine)
|
||||
}
|
||||
answer(again[0], composes)
|
||||
moved := ends(2, "the rerun")
|
||||
if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed ||
|
||||
!bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID {
|
||||
t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"])
|
||||
}
|
||||
|
||||
// A superseded step leaves the runs; its other reader goes on.
|
||||
step("step-r3a", "a", "r3", false)
|
||||
step("step-r3b", "b", "r3", false)
|
||||
sup := asks(2)
|
||||
readers(runKeyOf("run-r3", "n1"), 2)
|
||||
readers(runKeyOf("run-r3", "n2"), 2)
|
||||
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Module: "a", Branch: trunkBranch, Commit: commit, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Steps: []mpr.StepResult{}})
|
||||
if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, a := range sup {
|
||||
answer(a, composes)
|
||||
}
|
||||
if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed {
|
||||
t.Fatalf("the other reader: %+v", got)
|
||||
}
|
||||
select {
|
||||
case e := <-ended:
|
||||
t.Fatalf("a superseded step said its end: %+v", e)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
|
||||
// Restarted during a run: the new assigner takes it up by its ask.
|
||||
step("step-r4", "c", "r4", false)
|
||||
restart := asks(2)
|
||||
stop()
|
||||
for _, a := range restart {
|
||||
answer(a, composes)
|
||||
}
|
||||
from = b.journal.mark()
|
||||
b.startStoppable(t)
|
||||
b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second)
|
||||
if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 {
|
||||
t.Fatalf("the step taken up: %+v", e)
|
||||
}
|
||||
|
||||
// compositions-drop: the readers ended naming the drop, the key deleted.
|
||||
step("step-r5", "a", "r5", false)
|
||||
asks(2)
|
||||
body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"})
|
||||
reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second)
|
||||
if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) {
|
||||
t.Fatalf("compositions-drop answered %s %v", reply.Data, err)
|
||||
}
|
||||
if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") {
|
||||
t.Fatalf("a reader of the dropped run: %+v", e)
|
||||
}
|
||||
if keys := runKeys("run-r5"); len(keys) != 1 {
|
||||
t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys)
|
||||
}
|
||||
|
||||
// At the cap of 1 MiB no run starts: the step ends could not run, naming the fill.
|
||||
for i := range 2 {
|
||||
if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
step("step-r6", "a", "r6", false)
|
||||
if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") {
|
||||
t.Fatalf("at the cap: %+v", e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,642 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// runsMem is the state composition-runs in memory, its bound keys' times to live kept beside.
|
||||
type runsMem struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
ttl map[string]time.Duration
|
||||
bytes int64
|
||||
n uint64
|
||||
// beforeDel, when set, runs once before the next delete: another write landing between a read and the delete.
|
||||
beforeDel func()
|
||||
// beforeArm, when set, runs once before the next arm: another write landing between a read and the arm.
|
||||
beforeArm func()
|
||||
}
|
||||
|
||||
func (k *runsMem) another() {
|
||||
k.mu.Lock()
|
||||
f := k.beforeDel
|
||||
k.beforeDel = nil
|
||||
k.mu.Unlock()
|
||||
if f != nil {
|
||||
f()
|
||||
}
|
||||
}
|
||||
|
||||
func newRunsMem() *runsMem {
|
||||
return &runsMem{vals: map[string][]byte{}, revs: map[string]uint64{}, ttl: map[string]time.Duration{}}
|
||||
}
|
||||
|
||||
func (k *runsMem) get(key string) ([]byte, uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.vals[key], k.revs[key], nil
|
||||
}
|
||||
|
||||
func (k *runsMem) create(key string, v []byte) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.vals[key]; ok {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
k.n++
|
||||
k.vals[key], k.revs[key] = v, k.n
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.revs[key] != rev {
|
||||
return 0, errRunChanged
|
||||
}
|
||||
k.n++
|
||||
k.vals[key], k.revs[key] = v, k.n
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) delAt(key string, rev uint64) error {
|
||||
k.another()
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.vals[key]; !ok {
|
||||
return nil
|
||||
}
|
||||
if k.revs[key] != rev {
|
||||
return errRunChanged
|
||||
}
|
||||
delete(k.vals, key)
|
||||
delete(k.revs, key)
|
||||
delete(k.ttl, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *runsMem) keys() ([]string, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
var out []string
|
||||
for key := range k.vals {
|
||||
out = append(out, key)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
k.mu.Lock()
|
||||
f := k.beforeArm
|
||||
k.beforeArm = nil
|
||||
k.mu.Unlock()
|
||||
if f != nil {
|
||||
f()
|
||||
}
|
||||
raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.n++
|
||||
k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl
|
||||
return k.n, nil
|
||||
}
|
||||
|
||||
// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it.
|
||||
func (k *runsMem) expire(key string) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
delete(k.vals, key)
|
||||
delete(k.revs, key)
|
||||
delete(k.ttl, key)
|
||||
}
|
||||
|
||||
func (k *runsMem) used() (int64, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.bytes > 0 {
|
||||
return k.bytes, nil
|
||||
}
|
||||
var n int64
|
||||
for _, v := range k.vals {
|
||||
n += int64(len(v))
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Design 47 G §718, §719, §720, A-468, A-469d, A-469x: a run's key is `<candidate key>.<machine>`, each part escaped; two
|
||||
// steps of one candidate joining at once start one run and the second reads it; a run that could not run is replaced; at the
|
||||
// cap no run starts, naming the fill; a run is deleted composition-runs-kept after its end; the settings are read from their files.
|
||||
func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
step := func(id, module string) meshwants.CompositionReader {
|
||||
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: "main", Revision: 3})
|
||||
}
|
||||
t.Run("key", func(t *testing.T) {
|
||||
if k := runKeyOf("main.3", "n1"); k != "main=2e3.n1" {
|
||||
t.Errorf("the key is %q, want each part escaped as a key's name", k)
|
||||
}
|
||||
if isRunKey(runBoundPrefix+"main=2e3.n1") || isRunKey(runUntakenPrefix+"main=2e3.n1") || !isRunKey("main=2e3.n1") {
|
||||
t.Error("a bound key is taken for a run's or a run's for a bound key")
|
||||
}
|
||||
})
|
||||
t.Run("one run for two steps at once", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var wg sync.WaitGroup
|
||||
got := make([]joined, 2)
|
||||
for i, m := range []string{"a", "b"} {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
j, err := r.join(step("ask-"+m, m), "main.3", "n1", "run-"+m)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
got[i] = j
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
runKeys := func() []string {
|
||||
all, _ := kv.keys()
|
||||
return slices.DeleteFunc(all, func(k string) bool { return !isRunKey(k) })
|
||||
}
|
||||
if got[0].started == got[1].started {
|
||||
t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started)
|
||||
}
|
||||
keys := runKeys()
|
||||
run, _, _, _ := r.read(runKeyOf("main.3", "n1"))
|
||||
if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil {
|
||||
t.Errorf("keys %v, run %+v: want one key read by both", keys, run)
|
||||
}
|
||||
if j, _ := r.join(step("ask-a", "a"), "main.3", "n1", "run-c"); j.started || len(j.run.Readers) != 2 {
|
||||
t.Errorf("a reader joining again is added again or starts a run: %+v", j)
|
||||
}
|
||||
})
|
||||
t.Run("could not run is never read", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomeCouldNotRun
|
||||
return true
|
||||
})
|
||||
j, err := r.join(step("ask-b", "b"), "main.3", "n1", "run-2")
|
||||
if err != nil || !j.started || j.run.Ask != "run-2" || len(j.run.Readers) != 1 || j.run.Ended != nil {
|
||||
t.Errorf("after a could not run: %+v %v, want a new run read by its starter alone", j, err)
|
||||
}
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomePassed
|
||||
return true
|
||||
})
|
||||
if j, _ := r.join(step("ask-c", "c"), "main.3", "n1", "run-3"); j.started || j.run.Ask != "run-2" {
|
||||
t.Errorf("a passed run is not read: %+v", j)
|
||||
}
|
||||
if j, _ := r.join(step("ask-d", "d"), "main.4", "n1", "run-4"); !j.started {
|
||||
t.Errorf("a step of another candidate reads the first's run: %+v", j)
|
||||
}
|
||||
})
|
||||
t.Run("cap", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
kv.bytes = 16 << 20
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
_, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1")
|
||||
if err == nil || !strings.Contains(err.Error(), "at its cap of 16 MiB (16777216 bytes kept)") {
|
||||
t.Errorf("at the cap: %v, want the fill named", err)
|
||||
}
|
||||
})
|
||||
t.Run("kept", func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
at := now
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return at }, t.Logf)
|
||||
for _, c := range []string{"main.1", "main.2"} {
|
||||
if _, err := r.join(step("ask-"+c, "a"), c, "n1", "run-"+c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := now.Add(-25 * time.Hour)
|
||||
_, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true })
|
||||
_, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute)
|
||||
recent := now.Add(-time.Hour)
|
||||
_, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true })
|
||||
if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys, _ := kv.keys()
|
||||
for _, k := range []string{runBoundPrefix, runUntakenPrefix} {
|
||||
if v, _, _ := kv.get(k + runKeyOf("main.1", "n1")); v != nil {
|
||||
t.Errorf("the key %s stays past its run", k+runKeyOf("main.1", "n1"))
|
||||
}
|
||||
}
|
||||
keys = slices.DeleteFunc(keys, func(k string) bool { return !isRunKey(k) })
|
||||
if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) {
|
||||
t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want)
|
||||
}
|
||||
})
|
||||
t.Run("a listing that misses a key being written", func(t *testing.T) {
|
||||
kv := &missingRuns{runsMem: newRunsMem()}
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.hide = runKeyOf("main.3", "n1")
|
||||
if key, _, _ := r.keyOfAsk("run-1"); key != "" {
|
||||
t.Fatalf("the stand's listing does not hide the key")
|
||||
}
|
||||
r.saw(kv.hide, true) // as the state's watch hands it
|
||||
if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide {
|
||||
t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key)
|
||||
}
|
||||
r.saw(kv.hide, false)
|
||||
if keys, _ := r.keys(); slices.Contains(keys, kv.hide) {
|
||||
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
|
||||
}
|
||||
})
|
||||
// No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run
|
||||
// replacing the key, and its untaken bound, stand.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
ended time.Time
|
||||
act func(r *compositionRuns, key string) error
|
||||
}{
|
||||
{"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }},
|
||||
{"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error {
|
||||
_, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil)
|
||||
return err
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
if !c.ended.IsZero() {
|
||||
_, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true })
|
||||
}
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound
|
||||
for {
|
||||
raw, rev, _ := kv.get(key)
|
||||
var run meshwants.CompositionRun
|
||||
_ = json.Unmarshal(raw, &run)
|
||||
if run.Ended == nil {
|
||||
at := now
|
||||
run.Ended, run.Outcome = &at, "failed"
|
||||
}
|
||||
fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers})
|
||||
if _, err := kv.update(key, fresh, rev); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
|
||||
}
|
||||
if err := c.act(r, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil {
|
||||
t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok)
|
||||
}
|
||||
if v, _, _ := kv.get(runUntakenPrefix + key); v == nil {
|
||||
t.Error("the untaken bound the rerun's run armed is deleted")
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("settings", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
|
||||
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "", "MESH_ASSIGNER_BLAME_TRIES_FILE": "5"}
|
||||
env := map[string]string{}
|
||||
for name, content := range files {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env[name] = p
|
||||
}
|
||||
s, err := runSettingsOf(func(k string) string { return env[k] })
|
||||
if err != nil || s.capBytes != 8<<20 || s.kept != 2*time.Hour || s.stepBound != 2*time.Second || s.untaken != 24*time.Hour || s.blameTries != 5 {
|
||||
t.Errorf("settings %+v %v", s, err)
|
||||
}
|
||||
_ = os.WriteFile(env["MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE"], []byte("30"), 0o600)
|
||||
if _, err := runSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "not a positive Go duration") {
|
||||
t.Errorf("a bound of no unit: %v", err)
|
||||
}
|
||||
if s, err := runSettingsOf(func(string) string { return "" }); err != nil || s != defaultRunSettings {
|
||||
t.Errorf("no files: %+v %v, want the defaults", s, err)
|
||||
}
|
||||
})
|
||||
raw, _ := json.Marshal(step("x", "a"))
|
||||
if !strings.Contains(string(raw), `"id":"x"`) {
|
||||
t.Errorf("a reader names no ask id: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §720, A-466s, A-468b, A-469y, A-470: a run's untaken bound is armed at its start and each reader said waiting for a
|
||||
// build node; ask-taken arms the bound of composition-step-bound and ends the waits ran; each step-progress arms it again; the
|
||||
// bound's expiry drops the ask and ends the run could not run naming the ask and the bound, a step-ended heard later ignored and
|
||||
// journaled; the untaken bound's expiry ends it naming run-untaken-bound, the waits superseded; the readers of a run with no end
|
||||
// are taken up at the start.
|
||||
func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
reader := func(id string) meshwants.CompositionReader {
|
||||
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id, Branch: "main", Revision: 3})
|
||||
}
|
||||
type said struct{ event, how, module string }
|
||||
setup := func(t *testing.T) (*runsMem, *compositionRuns, *[]said, *[]string, *[]string) {
|
||||
kv := newRunsMem()
|
||||
var saids []said
|
||||
var drops, journal []string
|
||||
var mu sync.Mutex
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, func(f string, a ...any) {
|
||||
mu.Lock()
|
||||
journal = append(journal, fmt.Sprintf(f, a...))
|
||||
mu.Unlock()
|
||||
})
|
||||
r.say = func(event string, body any, _ string) error {
|
||||
w := body.(meshwants.Wait)
|
||||
if w.On != waitOnBuildNode || w.Step != "pipeline" || w.Owner != meshwants.Seat || w.Node == "" {
|
||||
t.Errorf("the wait said %+v", w)
|
||||
}
|
||||
saids = append(saids, said{event, w.How, w.Module})
|
||||
return nil
|
||||
}
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
|
||||
return kv, r, &saids, &drops, &journal
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
t.Run("taken, progress, bound", func(t *testing.T) {
|
||||
kv, r, saids, drops, journal := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kv.ttl[runUntakenPrefix+key] != 24*time.Hour || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitHeld, "", "b"}}) {
|
||||
t.Fatalf("at the start: untaken %v, said %v", kv.ttl, *saids)
|
||||
}
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: "someone-else"}); err != nil || kv.ttl[runBoundPrefix+key] != 0 {
|
||||
t.Fatalf("another asker's ask-taken arms a bound: %v %v", kv.ttl, err)
|
||||
}
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, untaken, _ := kv.get(runUntakenPrefix + key)
|
||||
if kv.ttl[runBoundPrefix+key] != 30*time.Minute || untaken != 0 || len(*saids) != 4 || (*saids)[2].how != "ran" || (*saids)[3].how != "ran" {
|
||||
t.Fatalf("at ask-taken: %v, said %v", kv.ttl, *saids)
|
||||
}
|
||||
before := kv.revs[runBoundPrefix+key]
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before {
|
||||
t.Fatalf("a step-progress does not arm the bound again: %v", err)
|
||||
}
|
||||
kv.expire(runBoundPrefix + key)
|
||||
r.expired(runBoundPrefix + key)
|
||||
run, _, _, _ := r.read(key)
|
||||
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" ||
|
||||
len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") {
|
||||
t.Fatalf("past the bound: %+v, drops %v", run, *drops)
|
||||
}
|
||||
if got := judgeRun("a", run); got.couldNot != run.Said {
|
||||
t.Errorf("a reader of the run past its bound: %+v", got)
|
||||
}
|
||||
late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"}
|
||||
if err := r.heardEnd(late, func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _, _, _ := r.read(key); again.Said != run.Said || !slices.ContainsFunc(*journal, func(l string) bool {
|
||||
return strings.Contains(l, "step-ended of the run ask run-1 is ignored") && strings.Contains(l, key)
|
||||
}) {
|
||||
t.Errorf("a step-ended after the bound: %+v, journal %v", again, *journal)
|
||||
}
|
||||
if keys, _ := kv.keys(); !slices.Equal(keys, []string{key}) {
|
||||
t.Errorf("the bound keys stay past the run's end: %v", keys)
|
||||
}
|
||||
})
|
||||
t.Run("untaken", func(t *testing.T) {
|
||||
kv, r, saids, drops, _ := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.expire(runUntakenPrefix + key)
|
||||
r.expired(runUntakenPrefix + key)
|
||||
run, _, _, _ := r.read(key)
|
||||
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" ||
|
||||
len(*drops) != 1 || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitEnded, "superseded", "a"}}) {
|
||||
t.Errorf("untaken past its bound: %+v, drops %v, said %v", run, *drops, *saids)
|
||||
}
|
||||
r.expired(runUntakenPrefix + key) // a run ended takes nothing
|
||||
if len(*drops) != 1 {
|
||||
t.Errorf("an ended run is dropped again: %v", *drops)
|
||||
}
|
||||
})
|
||||
// A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is
|
||||
// journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.).
|
||||
rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed
|
||||
return func() {
|
||||
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true })
|
||||
kv.expire(runBoundPrefix + key)
|
||||
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
|
||||
}
|
||||
}
|
||||
taken := func(t *testing.T, r *compositionRuns) {
|
||||
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
act func(t *testing.T, kv *runsMem, r *compositionRuns)
|
||||
ended bool
|
||||
bound string // the ask the bound key holds, "" for none
|
||||
untaken string
|
||||
journal string
|
||||
}{
|
||||
{"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
kv.beforeArm = rerun(kv, r)
|
||||
taken(t, r)
|
||||
}, false, "", "run-2", "the ask-taken of the run ask run-1 is late"},
|
||||
{"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
kv.beforeArm = rerun(kv, r)
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
|
||||
{"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
rerun(kv, r)()
|
||||
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
|
||||
{"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"},
|
||||
{"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.expire(runBoundPrefix + key)
|
||||
r.expired(runBoundPrefix + key)
|
||||
}, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"},
|
||||
{"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
|
||||
kv.expire(runUntakenPrefix + key)
|
||||
r.expired(runUntakenPrefix + key)
|
||||
}, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"},
|
||||
{"an end of another ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
if ended, err := r.end(key, "run-0", meshwants.CompositionRunOutcomePassed, "", nil); ended || err != nil {
|
||||
t.Errorf("the end of run-0 ended the run of run-1: %v %v", ended, err)
|
||||
}
|
||||
}, false, "", "run-1", ""},
|
||||
{"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
|
||||
taken(t, r)
|
||||
}, false, "run-1", "", ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
kv, r, _, _, journal := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.act(t, kv, r)
|
||||
run, _, _, _ := r.read(key)
|
||||
if (run.Ended != nil) != c.ended {
|
||||
t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended)
|
||||
}
|
||||
for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} {
|
||||
if held, _, _, _ := r.boundOf(prefix + key); held != want {
|
||||
t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want)
|
||||
}
|
||||
}
|
||||
if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) {
|
||||
t.Errorf("the journal %v does not say %q", *journal, c.journal)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A-469y, A-470u: a step asked again that joins a running run not yet taken, beside a reader or in its pipeline's former
|
||||
// reader's place, says its own wait held, as join does; the former reader's wait, and that of a reader leaving, ends at once
|
||||
// superseded.
|
||||
held := func(m string) said { return said{meshwants.EventWaitHeld, "", m} }
|
||||
gone := func(m string) said { return said{meshwants.EventWaitEnded, "superseded", m} }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
act func(t *testing.T, r *compositionRuns)
|
||||
want []said
|
||||
}{
|
||||
{"a rerun joins", func(t *testing.T, r *compositionRuns) {
|
||||
again := reader("b")
|
||||
again.Rerun = true
|
||||
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), held("b")}},
|
||||
{"a rerun takes its former place", func(t *testing.T, r *compositionRuns) {
|
||||
again := reader("a2")
|
||||
again.Pipeline, again.Module, again.Rerun = "main.3.a", "a2", true
|
||||
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), gone("a"), held("a2")}},
|
||||
{"a reader leaves", func(t *testing.T, r *compositionRuns) {
|
||||
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.leave("main.3.a", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}, []said{held("a"), held("b"), gone("a")}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
_, r, saids, _, _ := setup(t)
|
||||
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.act(t, r)
|
||||
if !slices.Equal(*saids, c.want) {
|
||||
t.Errorf("said %v, want %v", *saids, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("taken up at the start", func(t *testing.T) {
|
||||
_, r, _, _, _ := setup(t)
|
||||
for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} {
|
||||
if _, err := r.join(reader(c.id), "main.3", c.machine, "run-"+c.machine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ended := now
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &ended, "passed"; return true })
|
||||
// A rerun's run carrying readers whose steps said their end (A-470l, A-470m): c passed and is not taken up, d is marked rerun.
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n1"), func(run *meshwants.CompositionRun) bool {
|
||||
for _, id := range []string{"c", "d"} {
|
||||
c := reader(id)
|
||||
c.Ended, c.Rerun = &ended, id == "d"
|
||||
run.Readers = append(run.Readers, c)
|
||||
}
|
||||
return true
|
||||
})
|
||||
readers, err := r.readersOfOpen()
|
||||
if err != nil || len(readers) != 3 || readers[0].ID != "a" || readers[1].ID != "b" || readers[2].ID != "d" {
|
||||
t.Fatalf("the readers taken up: %+v %v", readers, err)
|
||||
}
|
||||
if keys, _ := r.keysOf("main.3", "a"); !slices.Equal(keys, []string{runKeyOf("main.3", "n1"), runKeyOf("main.3", "n2")}) {
|
||||
t.Errorf("the runs of reader a: %v", keys)
|
||||
}
|
||||
// A-470q: a reader with no `ended` says its first end under its ask's id; d, marked rerun, says its end again under
|
||||
// `<id>.<the open run's ask>`.
|
||||
ids := make(chan [2]string, 3)
|
||||
steps := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf,
|
||||
resume: func(context.Context, string, meshwants.StepRun) stepOutcome {
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
},
|
||||
say: func(_ string, body any, id string) error {
|
||||
ids <- [2]string{body.(meshwants.StepEnded).ID, id}
|
||||
return nil
|
||||
}}
|
||||
steps.takeUp(context.Background(), r)
|
||||
got := map[string]string{}
|
||||
for range 3 {
|
||||
select {
|
||||
case e := <-ids:
|
||||
got[e[0]] = e[1]
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the ends of the take-up: %v", got)
|
||||
}
|
||||
}
|
||||
if want := map[string]string{"a": "a", "b": "b", "d": "d.run-n1"}; !maps.Equal(got, want) {
|
||||
t.Errorf("the message ids of the take-up's ends: %v, want %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment.
|
||||
type missingRuns struct {
|
||||
*runsMem
|
||||
hide string
|
||||
}
|
||||
|
||||
func (k *missingRuns) keys() ([]string, error) {
|
||||
all, err := k.runsMem.keys()
|
||||
return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The composition runs' read and drop (design 47 G §728; A-469f, A-469u, A-469v): the seat mesh-wants' verbs `compositions`
|
||||
// (class read) and `compositions-drop` (class act, §391's default callers), answered by the assigner's process, which keeps the
|
||||
// runs and says the readers' ends; the module's tools `assigner_compositions` and `assigner_compositions_drop` call them.
|
||||
|
||||
// The tools that call the two verbs.
|
||||
const (
|
||||
compositionsTool = "assigner_compositions"
|
||||
compositionsDropTool = "assigner_compositions_drop"
|
||||
)
|
||||
|
||||
// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the
|
||||
// machine named only those, and the state's fill against its cap.
|
||||
func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) {
|
||||
all, err := r.keys()
|
||||
if err != nil {
|
||||
return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err)
|
||||
}
|
||||
slices.Sort(all)
|
||||
out := meshwants.CompositionsAnswer{Runs: []meshwants.CompositionRun{}}
|
||||
for _, key := range all {
|
||||
if !isRunKey(key) {
|
||||
continue
|
||||
}
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil || !ok || (in.Candidate != "" && run.Candidate != in.Candidate) || (in.Machine != "" && run.Machine != in.Machine) {
|
||||
continue
|
||||
}
|
||||
run.Key = key
|
||||
out.Runs = append(out.Runs, run)
|
||||
}
|
||||
out.Fill, err = r.fill()
|
||||
return out, err
|
||||
}
|
||||
|
||||
// drop drops one composition run by hand (§728): the act and why written in the hand-act log first (record), then a running run's
|
||||
// ask dropped, each of its readers' waits stopped and, for each reader whose end is owed (no `ended`, or marked rerun), its end
|
||||
// said could not run naming the drop, once (A-470r), and the key deleted with its bound keys. It answers the entry deleted and
|
||||
// the readers it ended.
|
||||
func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclient.Request) error, caller string, in meshwants.CompositionsDropInput) (meshwants.CompositionsDropAnswer, error) {
|
||||
why := strings.TrimSpace(in.Why)
|
||||
if in.Candidate == "" || in.Machine == "" || why == "" {
|
||||
return meshwants.CompositionsDropAnswer{}, errors.New("name the candidate, the machine and why the composition run is dropped")
|
||||
}
|
||||
key := runKeyOf(in.Candidate, in.Machine)
|
||||
run, _, ok, err := r.read(key)
|
||||
if err != nil {
|
||||
return meshwants.CompositionsDropAnswer{}, err
|
||||
}
|
||||
if !ok {
|
||||
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("no composition run of %s on %s is kept", in.Candidate, in.Machine)
|
||||
}
|
||||
if err := record(meshhandactsclient.Request{What: meshwants.VerbCompositionsDrop + " " + key, Why: why + " (" + caller + ")", Cause: "compositions-drop"}); err != nil {
|
||||
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
|
||||
}
|
||||
said := fmt.Sprintf("%s: the composition run %s was dropped by hand: %s", in.Machine, key, why)
|
||||
answer := meshwants.CompositionsDropAnswer{Run: run}
|
||||
answer.Run.Key = key
|
||||
if run.Ended == nil {
|
||||
if r.drop != nil {
|
||||
if err := r.drop(run.Ask, said); err != nil {
|
||||
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
|
||||
}
|
||||
}
|
||||
ids := make([]string, 0, len(run.Readers))
|
||||
for _, reader := range run.Readers {
|
||||
ids = append(ids, reader.ID)
|
||||
}
|
||||
c.leave(ids) // their waits say no end of their own
|
||||
// The run ends without moving its readers: the drop says each owed end itself, once (A-470r).
|
||||
if _, err := r.endRunMoving(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, r.untaken(key, run.Ask), false); err != nil {
|
||||
return answer, err
|
||||
}
|
||||
shared := []meshwants.Shared{{Machine: run.Machine, Run: key, Ask: run.Ask}}
|
||||
for _, reader := range run.Readers {
|
||||
if reader.Ended != nil && !reader.Rerun {
|
||||
continue // it ended passed already and keeps its verdict (A-470r)
|
||||
}
|
||||
again := "" // a reader whose step said its end already says it again under the dropped run's ask (A-470q)
|
||||
if reader.Ended != nil {
|
||||
again = run.Ask
|
||||
}
|
||||
c.sayEnd(reader.ID, stepRunOf(reader), nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said, Shared: shared}, again)
|
||||
answer.Ended = append(answer.Ended, reader)
|
||||
}
|
||||
}
|
||||
// The dropped run's key goes while it holds that run: a run started on it since stands.
|
||||
err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask })
|
||||
r.heard()
|
||||
return answer, err
|
||||
}
|
||||
|
||||
// compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop.
|
||||
func compositionVerbs(r *compositionRuns, c *compositionSteps, record func(meshhandactsclient.Request) error) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
meshwants.VerbCompositions: func(_ string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.CompositionsInput
|
||||
if len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
}
|
||||
return r.compositions(in)
|
||||
},
|
||||
meshwants.VerbCompositionsDrop: func(caller string, raw json.RawMessage) (any, error) {
|
||||
var in meshwants.CompositionsDropInput
|
||||
if err := json.Unmarshal(raw, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
|
||||
}
|
||||
return c.drop(r, record, caller, in)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// compositionsTools are the module's tools of §728's names, each calling its verb (A-469f).
|
||||
func compositionsTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
|
||||
call := func(verb string, args map[string]any, names ...string) (any, error) {
|
||||
in := map[string]any{}
|
||||
for _, n := range names {
|
||||
if v := arg(args, n); v != "" {
|
||||
in[n] = v
|
||||
}
|
||||
}
|
||||
raw, err := ask("seat:"+meshwants.Seat+"."+verb, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s answered what is not JSON: %w", verb, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: compositionsTool, Description: "The composition runs the assigner keeps, one per candidate and machine (design 47 G §719, §728): each with " +
|
||||
"its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those. Read-only.",
|
||||
Input: schema(map[string]string{"candidate": "the candidate's key: <branch>.<revision>, or run-<id>", "machine": "a machine"}, nil),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return call(meshwants.VerbCompositions, args, "candidate", "machine")
|
||||
}},
|
||||
{Name: compositionsDropTool, Description: "Drop one composition run by hand (design 47 G §728): written in the hand-act log first, then its ask " +
|
||||
"is dropped, each reader ended could not run naming the drop, and the key deleted.",
|
||||
Input: schema(map[string]string{"candidate": "the candidate's key", "machine": "the machine", "why": "why the run is dropped"}, []string{"candidate", "machine", "why"}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return call(meshwants.VerbCompositionsDrop, args, "candidate", "machine", "why")
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 G §728, A-469f, A-469u, A-469v: compositions answers each run with its key and the fill, only those of a candidate or
|
||||
// machine when named; compositions-drop writes the hand act first (nothing is dropped when it is not recorded), drops a running
|
||||
// run's ask, ends each reader could not run naming the drop and deletes the key with its bound keys; the tools call the verbs.
|
||||
func TestTheCompositionRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
kv := newRunsMem()
|
||||
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var drops []string
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask); return nil }
|
||||
ends := newSaidEvents()
|
||||
var endIDs []string // the message id of each end said
|
||||
c := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf, say: func(event string, body any, id string) error {
|
||||
endIDs = append(endIDs, id)
|
||||
return ends.say(event, body, id)
|
||||
}}
|
||||
moves := 0
|
||||
move := func(meshwants.CompositionRun) { moves++ }
|
||||
r.moved.Store(&move)
|
||||
for _, m := range []string{"n1", "n2"} {
|
||||
if _, err := r.join(readerOf("ask-a", meshwants.StepRun{Pipeline: "main.3.a", Step: meshwants.StepRunStepComposition, Module: "a"}), "main.3", m, "run-"+m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A-470r: n2's run carries a reader that ended passed (p) and one marked rerun (q); only q's end is owed, said once.
|
||||
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(run *meshwants.CompositionRun) bool {
|
||||
for _, id := range []string{"ask-p", "ask-q"} {
|
||||
c := readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id})
|
||||
c.Ended, c.Rerun = &now, id == "ask-q"
|
||||
run.Readers = append(run.Readers, c)
|
||||
}
|
||||
return true
|
||||
})
|
||||
var recorded []meshhandactsclient.Request
|
||||
record := func(q meshhandactsclient.Request) error { recorded = append(recorded, q); return nil }
|
||||
verbs := compositionVerbs(r, c, record)
|
||||
out, err := verbs[meshwants.VerbCompositions]("op", json.RawMessage(`{"machine":"n2"}`))
|
||||
read, _ := out.(meshwants.CompositionsAnswer)
|
||||
if err != nil || len(read.Runs) != 1 || read.Runs[0].Key != runKeyOf("main.3", "n2") || read.Runs[0].Ask != "run-n2" || len(read.Runs[0].Readers) != 3 ||
|
||||
read.Fill.Store != "composition-runs" || read.Fill.Cap != 16<<20 || read.Fill.Bytes == 0 {
|
||||
t.Fatalf("compositions = %+v %v", out, err)
|
||||
}
|
||||
refuse := compositionVerbs(r, c, func(meshhandactsclient.Request) error { return errors.New("no holder") })
|
||||
if _, err := refuse[meshwants.VerbCompositionsDrop]("op", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "nothing was dropped") || len(drops) != 0 {
|
||||
t.Fatalf("a drop not recorded: %v, drops %v", err, drops)
|
||||
}
|
||||
out, err = verbs[meshwants.VerbCompositionsDrop]("op@n1", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`))
|
||||
dropped, _ := out.(meshwants.CompositionsDropAnswer)
|
||||
if err != nil || dropped.Run.Key != runKeyOf("main.3", "n2") || len(dropped.Ended) != 2 || dropped.Ended[0].ID != "ask-a" || dropped.Ended[1].ID != "ask-q" ||
|
||||
moves != 0 || len(endIDs) != 2 || endIDs[0] != "ask-a" || endIDs[1] != "ask-q.run-n2" ||
|
||||
len(recorded) != 1 || recorded[0].Why != "stuck (op@n1)" || len(drops) != 1 || drops[0] != "run-n2" {
|
||||
t.Fatalf("compositions-drop = %+v %v, recorded %+v, drops %v, moves %d, ends %v", out, err, recorded, drops, moves, endIDs)
|
||||
}
|
||||
end, _ := ends.said[meshwants.EventStepEnded+".n1"].(meshwants.StepEnded)
|
||||
if end.ID != "ask-q" || end.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "dropped by hand: stuck") ||
|
||||
len(end.Shared) != 1 || end.Shared[0].Ask != "run-n2" {
|
||||
t.Errorf("the reader's end %+v", end)
|
||||
}
|
||||
if keys, _ := kv.keys(); len(keys) != 2 || keys[0] != runKeyOf("main.3", "n1") {
|
||||
t.Errorf("keys after the drop %v, want n1's run and its untaken key", keys)
|
||||
}
|
||||
// The tools call the verbs with what they were given.
|
||||
var asked []string
|
||||
tools := compositionsTools(func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
asked = append(asked, key+" "+string(raw))
|
||||
return json.RawMessage(`{"runs":[],"fill":{"store":"composition-runs","bytes":0,"cap":1}}`), nil
|
||||
})
|
||||
for _, tool := range tools {
|
||||
if _, err := tool.Run(map[string]any{"candidate": "main.3", "machine": "n1", "why": "x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(tools) != 2 || tools[0].Name != compositionsTool || tools[1].Name != compositionsDropTool ||
|
||||
asked[0] != `seat:mesh-wants.compositions {"candidate":"main.3","machine":"n1"}` || asked[1] != `seat:mesh-wants.compositions-drop {"candidate":"main.3","machine":"n1","why":"x"}` {
|
||||
t.Errorf("the tools asked %v", asked)
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,8 @@ type stepOutcome struct {
|
||||
Outcome string
|
||||
Said string
|
||||
Warnings []string
|
||||
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
|
||||
Shared []meshwants.Shared
|
||||
}
|
||||
|
||||
// compositionSteps takes the composition steps and says their ends.
|
||||
@@ -30,14 +32,21 @@ type compositionSteps struct {
|
||||
node string
|
||||
// say says one event of the seat mesh-wants (wantsPublisher).
|
||||
say publisher
|
||||
// run runs one step and answers what it came to.
|
||||
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
// run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp).
|
||||
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
|
||||
// ended writes, once the step asked under id said its end, that time on its reader in each run naming it (A-470m); nil
|
||||
// writes none.
|
||||
ended func(id string, run meshwants.StepRun, at time.Time)
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
// asker is the name the step's progress names as its asker, read when it is said (stepAskerOf).
|
||||
asker func() string
|
||||
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
|
||||
ticks func(d time.Duration) (<-chan time.Time, func())
|
||||
mu sync.Mutex
|
||||
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave).
|
||||
stops map[string]context.CancelFunc
|
||||
}
|
||||
|
||||
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
|
||||
@@ -69,10 +78,27 @@ func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
|
||||
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
|
||||
const progressEvery = time.Minute
|
||||
|
||||
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context.
|
||||
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
|
||||
// composition runs it waits on (§723, §729).
|
||||
type stepPhase struct {
|
||||
mu sync.Mutex
|
||||
phase string
|
||||
mu sync.Mutex
|
||||
phase string
|
||||
shared []meshwants.Shared
|
||||
}
|
||||
|
||||
// setShared names the runs the step ctx runs waits on, when it runs one.
|
||||
func setShared(ctx context.Context, shared []meshwants.Shared) {
|
||||
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
|
||||
p.mu.Lock()
|
||||
p.shared = shared
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *stepPhase) waitsOn() []meshwants.Shared {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.shared
|
||||
}
|
||||
|
||||
type phaseKey struct{}
|
||||
@@ -100,23 +126,97 @@ func (c *compositionSteps) take(ctx context.Context, id string, data []byte) boo
|
||||
c.logf("a step ask that is no composition step with an id is left: %v", err)
|
||||
return false
|
||||
}
|
||||
go c.runOne(ctx, id, run)
|
||||
go c.say1(ctx, id, run, c.run, "")
|
||||
return true
|
||||
}
|
||||
|
||||
// runOne runs one step and says its end.
|
||||
func (c *compositionSteps) runOne(ctx context.Context, id string, run meshwants.StepRun) {
|
||||
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
|
||||
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
|
||||
// says nothing: the runner ended it already.
|
||||
func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
|
||||
started := c.now().UTC()
|
||||
ctx, stopWait := context.WithCancel(ctx)
|
||||
c.mu.Lock()
|
||||
if c.stops == nil {
|
||||
c.stops = map[string]context.CancelFunc{}
|
||||
}
|
||||
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
|
||||
c.mu.Unlock()
|
||||
stopWait()
|
||||
return
|
||||
}
|
||||
c.stops[id] = stopWait
|
||||
c.mu.Unlock()
|
||||
defer func() {
|
||||
c.mu.Lock()
|
||||
delete(c.stops, id)
|
||||
c.mu.Unlock()
|
||||
stopWait()
|
||||
}()
|
||||
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
|
||||
stop := c.sayProgress(id, phase)
|
||||
o := c.run(context.WithValue(ctx, phaseKey{}, phase), id, run)
|
||||
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
|
||||
stop()
|
||||
if ctx.Err() != nil {
|
||||
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
|
||||
return
|
||||
}
|
||||
c.sayEnd(id, run, &started, o, again)
|
||||
}
|
||||
|
||||
// sayEnd says the end of the step asked under id as the seat's step-ended.<node>, its id the ask's; once said, its time is
|
||||
// written as its reader's `ended` (A-470m). Its first end's message id is the ask's id; an end said again is published under
|
||||
// `<ask id>.<again>`, again the ask of the run whose end makes it say again, so the bus's deduplication never drops it (A-470q).
|
||||
func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *time.Time, o stepOutcome, again string) {
|
||||
ended := c.now().UTC()
|
||||
if started == nil {
|
||||
started = &ended
|
||||
}
|
||||
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module,
|
||||
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
|
||||
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended}
|
||||
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil {
|
||||
Rerun: run.Rerun, By: run.By, Started: started, Ended: &ended, Shared: o.Shared}
|
||||
msgID := id
|
||||
if again != "" {
|
||||
msgID = id + "." + again
|
||||
}
|
||||
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
|
||||
c.logf("saying the end of the composition step %s: %v", id, err)
|
||||
return
|
||||
}
|
||||
if c.ended != nil {
|
||||
c.ended(id, run, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged
|
||||
// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end.
|
||||
func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) {
|
||||
return func(run meshwants.CompositionRun) {
|
||||
for _, reader := range run.Readers {
|
||||
if !reader.Rerun || c.running(reader.ID) {
|
||||
continue
|
||||
}
|
||||
c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask)
|
||||
go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume, run.Ask) // said again under the moved run's ask (A-470q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// running is whether the step asked under id runs here.
|
||||
func (c *compositionSteps) running(id string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.stops[id] != nil
|
||||
}
|
||||
|
||||
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
|
||||
func (c *compositionSteps) leave(ids []string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
if stop := c.stops[id]; stop != nil {
|
||||
stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,7 +239,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()
|
||||
return
|
||||
case <-tick:
|
||||
at := c.now().UTC()
|
||||
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at}
|
||||
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()}
|
||||
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
|
||||
c.logf("saying the progress of the composition step %s: %v", id, err)
|
||||
}
|
||||
|
||||
+20
-4
@@ -168,7 +168,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
|
||||
t.Fatal("the composition step said no end")
|
||||
}
|
||||
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
|
||||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 1 || !strings.Contains(e.Warnings[0], "n3 is away") {
|
||||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 2 || !strings.Contains(e.Warnings[0], "n3 is away") ||
|
||||
e.Warnings[1] != "n2: the composer names no culprits" {
|
||||
t.Fatalf("the step ended %+v", e)
|
||||
}
|
||||
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
|
||||
@@ -180,9 +181,24 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
|
||||
t.Error("the machine away was asked for its input")
|
||||
}
|
||||
}
|
||||
keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx)
|
||||
if len(keys) != 0 {
|
||||
t.Errorf("the inputs stay kept past their step-ended: %v", keys)
|
||||
// The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch.
|
||||
inputs := b.buckets[buckets.Inputs()]
|
||||
w, err := inputs.WatchAll(b.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for {
|
||||
keys, _ := inputs.Keys(b.ctx)
|
||||
if len(keys) == 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-w.Updates():
|
||||
case <-deadline:
|
||||
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -40,7 +42,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
|
||||
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
|
||||
asked = run
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
|
||||
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
|
||||
}}
|
||||
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
|
||||
if c.take(context.Background(), "ask-0", other) {
|
||||
@@ -67,7 +70,8 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
|
||||
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
|
||||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
|
||||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
|
||||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil {
|
||||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil ||
|
||||
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
|
||||
t.Fatalf("step-ended %+v", end)
|
||||
}
|
||||
}
|
||||
@@ -105,6 +109,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
},
|
||||
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
|
||||
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
|
||||
setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}})
|
||||
<-release
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}}
|
||||
@@ -122,7 +127,8 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
said.mu.Lock()
|
||||
p, ok := said.said[progress].(meshwants.StepProgressEvent)
|
||||
said.mu.Unlock()
|
||||
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) {
|
||||
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) ||
|
||||
len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723)
|
||||
t.Fatalf("step-progress %+v", p)
|
||||
}
|
||||
close(release)
|
||||
@@ -135,3 +141,70 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
|
||||
t.Fatal("the ticker is not stopped when the step has ended")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end;
|
||||
// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no
|
||||
// reader is left", so a later step starts a new run.
|
||||
func TestASupersededStepLeavesItsRuns(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
|
||||
r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf)
|
||||
var drops []string
|
||||
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
|
||||
step := func(pipeline string) meshwants.StepRun {
|
||||
return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3}
|
||||
}
|
||||
said := newSaidEvents()
|
||||
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf,
|
||||
run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome {
|
||||
j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
r.await(ctx, []string{j.key})
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
}}
|
||||
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
|
||||
for _, p := range []string{"main.3.a", "main.3.b"} {
|
||||
if !c.take(context.Background(), "ask-"+p, body(step(p))) {
|
||||
t.Fatal("not taken")
|
||||
}
|
||||
}
|
||||
key := runKeyOf("main.3", "n1")
|
||||
for {
|
||||
wake := r.wake()
|
||||
if run, _, _, _ := r.read(key); len(run.Readers) == 2 {
|
||||
break
|
||||
}
|
||||
<-wake
|
||||
}
|
||||
ids, err := r.leave("main.3.a", c.leave)
|
||||
run, _, _, _ := r.read(key)
|
||||
if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 {
|
||||
t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops)
|
||||
}
|
||||
_, err = r.leave("main.3.b", c.leave)
|
||||
run, _, _, _ = r.read(key)
|
||||
if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 ||
|
||||
!strings.HasPrefix(drops[0], run.Ask+": no reader is left") {
|
||||
t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops)
|
||||
}
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
n := len(c.stops)
|
||||
c.mu.Unlock()
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("the steps that left still wait")
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
said.mu.Lock()
|
||||
defer said.mu.Unlock()
|
||||
if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok {
|
||||
t.Errorf("a step that left says its end: %v", said.said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,14 +25,17 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
|
||||
// request it asked a build seat to run in its state `inputs`, one key `<time>.<digest>` per input, the value the input's bytes in
|
||||
// request it asked a build seat to run in its state `inputs`, one key per input, its sha256 alone (A-465a), the value the input's bytes in
|
||||
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
|
||||
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
|
||||
// An input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, whichever is first, the day
|
||||
// judged at each new ask over at most 20 of the oldest keys. `assigner_inputs` reads them. This mirrors the indexer's
|
||||
// An input is kept once per content: every ask that names its digest shares it (A-465a). It is deleted, with its object, when the
|
||||
// step-ended of the last ask that names it is heard, or a day after its last write, whichever is first, the day judged at each new
|
||||
// ask over at most 20 of the oldest inputs. `assigner_inputs` reads them. This mirrors the indexer's
|
||||
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
|
||||
|
||||
// inputsTool is the tool that reads the kept inputs.
|
||||
@@ -45,10 +48,39 @@ const (
|
||||
inputsSwept = 20
|
||||
inputsTime = "20060102T150405Z"
|
||||
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
|
||||
// holding the input's own key, so the build seat's step-ended of that ask finds it.
|
||||
// holding the digest of the input the ask uses, so the build seat's step-ended of that ask finds it.
|
||||
inputsAskPrefix = "ask."
|
||||
)
|
||||
|
||||
// inputAsk is what an ask's key holds (A-465b): the digest of the input the ask uses and the time the ask wrote it, in inputsTime.
|
||||
type inputAsk struct {
|
||||
Digest string `json:"digest"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
// readAsk reads an ask's key: the JSON form, or the bare key of the old form `<time>.<digest>` an earlier assigner wrote, which
|
||||
// carries its time in the key itself.
|
||||
func readAsk(raw []byte) (digest string, at time.Time) {
|
||||
var a inputAsk
|
||||
if json.Unmarshal(raw, &a) == nil && a.Digest != "" {
|
||||
at, _ = time.Parse(inputsTime, a.At)
|
||||
return a.Digest, at
|
||||
}
|
||||
digest = string(raw)
|
||||
t, _, _ := strings.Cut(digest, ".")
|
||||
at, _ = time.Parse(inputsTime, t)
|
||||
return digest, at
|
||||
}
|
||||
|
||||
// isDigest is whether a key is an input's own key: 64 hex digits.
|
||||
func isDigest(key string) bool {
|
||||
if len(key) != 64 {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// digestOf is an input's digest as the ask names it: its sha256 in hex.
|
||||
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
|
||||
|
||||
@@ -64,8 +96,9 @@ type inputKeeper struct {
|
||||
used func() (int64, error)
|
||||
}
|
||||
|
||||
// keep keeps the exact bytes of the input of the ask under their write time and sha256, after sweeping the keys more than a day old
|
||||
// among the 20 oldest, and names it by the ask's id; it answers the digest. The same bytes written again in the same second are one key.
|
||||
// keep keeps the exact bytes of the input of the ask under their sha256, after sweeping, and names the digest and the time in the
|
||||
// ask's key; it answers the digest. The same bytes asked again are the same one input (A-465a, A-465b). The ask's key is written
|
||||
// first, so a sweep never finds the input unnamed.
|
||||
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
|
||||
if ask == "" {
|
||||
return "", errors.New("name the ask the input is kept for")
|
||||
@@ -73,29 +106,43 @@ func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
|
||||
if err := k.sweep(); err != nil {
|
||||
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
|
||||
}
|
||||
if k.used != nil {
|
||||
used, err := k.used()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", meshwants.StateInputs, err)
|
||||
}
|
||||
if used+int64(len(body)) > inputsCap {
|
||||
return "", fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
|
||||
meshwants.StateInputs, inputsCap>>20, used, len(body))
|
||||
}
|
||||
}
|
||||
digest := digestOf(body)
|
||||
key := meshwants.InputsByTimeDigestKey(k.now().UTC().Format(inputsTime), digest)
|
||||
if err := k.vals.Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
raw, err := json.Marshal(inputAsk{Digest: digest, At: k.now().UTC().Format(inputsTime)})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := k.vals.KV.Create(inputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
askKey := inputsAskPrefix + keys.EscapeName(ask)
|
||||
if err := k.vals.KV.Create(askKey, raw); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
return "", err
|
||||
}
|
||||
have, err := k.vals.KV.Get(digest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if have != nil {
|
||||
return digest, nil
|
||||
}
|
||||
if k.used != nil {
|
||||
used, err := k.used()
|
||||
if err == nil && used+int64(len(body)) > inputsCap {
|
||||
err = fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
|
||||
buckets.InputsState, inputsCap>>20, used, len(body))
|
||||
} else if err != nil {
|
||||
err = fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", buckets.InputsState, err)
|
||||
}
|
||||
if err != nil {
|
||||
_ = k.vals.KV.Delete(askKey)
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := k.vals.Put(digest, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// ended deletes the input kept for an ask when the build seat's step-ended of that ask is heard; an ask that kept none, or whose
|
||||
// input a sweep deleted, changes nothing.
|
||||
// ended removes the ask's key when the build seat's step-ended of that ask is heard, and deletes the input it named, with its
|
||||
// object, when no other ask's key names it (A-465a); an ask that kept none changes nothing.
|
||||
func (k *inputKeeper) ended(ask string) error {
|
||||
if ask == "" {
|
||||
return nil
|
||||
@@ -105,10 +152,41 @@ func (k *inputKeeper) ended(ask string) error {
|
||||
if err != nil || e == nil {
|
||||
return err
|
||||
}
|
||||
if err := k.deleteInput(string(e.Value)); err != nil {
|
||||
digest, _ := readAsk(e.Value)
|
||||
if err := k.vals.KV.Delete(askKey); err != nil {
|
||||
return err
|
||||
}
|
||||
return k.vals.KV.Delete(askKey)
|
||||
named, err := k.named()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if named[digest] {
|
||||
return nil // another running ask still names it
|
||||
}
|
||||
return k.deleteInput(digest)
|
||||
}
|
||||
|
||||
// named is the inputs some ask's key names now.
|
||||
func (k *inputKeeper) named() (map[string]bool, error) {
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
named := map[string]bool{}
|
||||
for _, key := range all {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
continue
|
||||
}
|
||||
o, err := k.vals.KV.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if o != nil {
|
||||
d, _ := readAsk(o.Value)
|
||||
named[d] = true
|
||||
}
|
||||
}
|
||||
return named, nil
|
||||
}
|
||||
|
||||
// deleteInput deletes an input's key and the object that key names, when it names one (A-465): an owner that ends its own key
|
||||
@@ -163,60 +241,76 @@ func (k *inputKeeper) dropAtStart() {
|
||||
k.logf("inputs: %d objects of deleted keys removed at the start, %d bytes", len(dropped), bytes)
|
||||
}
|
||||
|
||||
// sweep deletes, among the 20 oldest keys, those written more than a day ago, and a name by ask whose input is gone.
|
||||
// sweep deletes, among the 20 oldest ask keys, those written more than a day ago; every input of the digest form no ask's key names
|
||||
// now, with its object; a key of the old form `<time>.<digest>` a day after the time it carries; and an old-form ask key whose
|
||||
// input is gone (A-465b).
|
||||
func (k *inputKeeper) sweep() error {
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
var inputs, asks []string
|
||||
type askAt struct {
|
||||
key string
|
||||
at time.Time
|
||||
}
|
||||
var asks []askAt
|
||||
var inputs []string
|
||||
for _, key := range all {
|
||||
if strings.HasPrefix(key, inputsAskPrefix) {
|
||||
asks = append(asks, key)
|
||||
} else {
|
||||
inputs = append(inputs, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(inputs) // the write time leads the key, so the oldest come first
|
||||
if len(inputs) > inputsSwept {
|
||||
inputs = inputs[:inputsSwept]
|
||||
}
|
||||
for _, key := range inputs {
|
||||
t, _, ok := meshwants.ParseInputsByTimeDigestKey(key)
|
||||
at, perr := time.Parse(inputsTime, t)
|
||||
if !ok || perr != nil || k.now().Sub(at) < inputsKeptFor {
|
||||
continue
|
||||
}
|
||||
if err := k.deleteInput(key); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
for _, key := range asks {
|
||||
if e, err := k.vals.KV.Get(key); err == nil && e != nil {
|
||||
if gone, err := k.vals.KV.Get(string(e.Value)); err == nil && gone == nil {
|
||||
if err := k.vals.KV.Delete(key); err != nil {
|
||||
errs = append(errs, err)
|
||||
e, err := k.vals.KV.Get(key)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
d, at := readAsk(e.Value)
|
||||
if !isDigest(d) {
|
||||
if gone, err := k.vals.KV.Get(d); err == nil && gone == nil {
|
||||
errs = append(errs, k.vals.KV.Delete(key))
|
||||
}
|
||||
}
|
||||
asks = append(asks, askAt{key, at})
|
||||
continue
|
||||
}
|
||||
if !isDigest(key) {
|
||||
t, _, _ := strings.Cut(key, ".")
|
||||
if at, err := time.Parse(inputsTime, t); err == nil && k.now().Sub(at) >= inputsKeptFor {
|
||||
errs = append(errs, k.deleteInput(key))
|
||||
}
|
||||
continue
|
||||
}
|
||||
inputs = append(inputs, key)
|
||||
}
|
||||
slices.SortFunc(asks, func(a, b askAt) int { return a.at.Compare(b.at) })
|
||||
if len(asks) > inputsSwept {
|
||||
asks = asks[:inputsSwept]
|
||||
}
|
||||
for _, a := range asks {
|
||||
if !a.at.IsZero() && k.now().Sub(a.at) >= inputsKeptFor {
|
||||
errs = append(errs, k.vals.KV.Delete(a.key))
|
||||
}
|
||||
}
|
||||
named, err := k.named()
|
||||
if err != nil {
|
||||
return errors.Join(append(errs, err)...)
|
||||
}
|
||||
for _, key := range inputs {
|
||||
if !named[key] {
|
||||
errs = append(errs, k.deleteInput(key))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// keyOf is the key of the input of the digest, or none.
|
||||
// keyOf is the key of the input of the digest, or none: the digest itself.
|
||||
func (k *inputKeeper) keyOf(digest string) (string, error) {
|
||||
all, err := k.vals.KV.Keys()
|
||||
if err != nil {
|
||||
if !isDigest(digest) {
|
||||
return "", nil
|
||||
}
|
||||
e, err := k.vals.KV.Get(digest)
|
||||
if err != nil || e == nil {
|
||||
return "", err
|
||||
}
|
||||
slices.Sort(all)
|
||||
for _, key := range all {
|
||||
if _, d, ok := meshwants.ParseInputsByTimeDigestKey(key); ok && d == digest && !strings.HasPrefix(key, inputsAskPrefix) {
|
||||
return key, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
|
||||
@@ -226,7 +320,7 @@ func (k *inputKeeper) body(digest string) ([]byte, error) {
|
||||
}
|
||||
key, err := k.keyOf(digest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state %s cannot be read: %w", meshwants.StateInputs, err)
|
||||
return nil, fmt.Errorf("the state %s cannot be read: %w", buckets.InputsState, err)
|
||||
}
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
|
||||
@@ -294,7 +388,7 @@ func (o *inputObjects) open() (stateobjects.Objects, error) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(meshwants.StateInputs)))
|
||||
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(buckets.InputsState)))
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, fmt.Errorf("the object store of the state inputs is not on the bus yet: %w", process.ErrNotYet)
|
||||
}
|
||||
@@ -345,7 +439,7 @@ func inputsOver(kv jetstream.KeyValue, js jetstream.JetStream, module string, lo
|
||||
return nil
|
||||
}
|
||||
objects := &inputObjects{js: js, module: module}
|
||||
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(meshwants.StateInputs)}
|
||||
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(buckets.InputsState)}
|
||||
used := func() (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
@@ -386,12 +480,21 @@ func inputsRead(k *inputKeeper) stdio.Tool {
|
||||
return nil, fmt.Errorf("the inputs cannot be listed: %w", err)
|
||||
}
|
||||
slices.Sort(all)
|
||||
lastAsk := map[string]string{} // the latest write time an ask's key names for each input
|
||||
for _, key := range all {
|
||||
if e, err := k.vals.KV.Get(key); err == nil && e != nil && strings.HasPrefix(key, inputsAskPrefix) {
|
||||
if d, at := readAsk(e.Value); !at.IsZero() && at.Format(inputsTime) > lastAsk[d] {
|
||||
lastAsk[d] = at.Format(inputsTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
out := []inputSaid{}
|
||||
for _, key := range all {
|
||||
t, d, ok := meshwants.ParseInputsByTimeDigestKey(key)
|
||||
if !ok || strings.HasPrefix(key, inputsAskPrefix) || (want != "" && d != want) {
|
||||
d := key
|
||||
if !isDigest(key) || (want != "" && d != want) {
|
||||
continue
|
||||
}
|
||||
t := lastAsk[key]
|
||||
var text string
|
||||
if found, err := k.vals.Get(key, &text); err != nil || !found {
|
||||
continue
|
||||
@@ -451,22 +554,28 @@ func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error))
|
||||
}}
|
||||
}
|
||||
|
||||
// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the
|
||||
// composition run that waits for that ask (A-240); one whose input cannot be deleted is given again by the bus.
|
||||
func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(string, ...any)) {
|
||||
// heardStepEnded hands the end a build seat's step-ended names to the composition run of that ask (A-240, §719), which writes it
|
||||
// in the run's key, and then deletes the input kept for the ask (design 47 G §354): the run's end first, so no other act delays
|
||||
// it past its bound (A-470o, A-470p). One whose run's end cannot be written, or whose input cannot be deleted, is given again by
|
||||
// the bus; its redelivery finds the run ended, said as ignored for the run, and still deletes the input.
|
||||
func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var e buildstep.StepEnded
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if runEnded != nil {
|
||||
if err := runEnded(e); err != nil {
|
||||
logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := k.ended(e.Ask); err != nil {
|
||||
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
if ends != nil {
|
||||
ends.heard(e)
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
)
|
||||
@@ -23,6 +25,8 @@ type kvMem struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
// delErr, when set, fails the next Delete once.
|
||||
delErr error
|
||||
}
|
||||
|
||||
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
@@ -57,6 +61,10 @@ func (k *kvMem) Update(key string, v []byte, rev uint64) error {
|
||||
func (k *kvMem) Delete(key string) error {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if err := k.delErr; err != nil {
|
||||
k.delErr = nil
|
||||
return err
|
||||
}
|
||||
delete(k.vals, key)
|
||||
return nil
|
||||
}
|
||||
@@ -246,6 +254,47 @@ func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
|
||||
if _, err := k.body(d2); err == nil {
|
||||
t.Error("an input a day old is still kept at the next ask")
|
||||
}
|
||||
// A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again,
|
||||
// whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
delErr error
|
||||
runErr error
|
||||
naked bool
|
||||
runs int
|
||||
deleted bool
|
||||
}{
|
||||
{name: "the run's end before the input's delete", runs: 1, deleted: true},
|
||||
{name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true},
|
||||
{name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`))
|
||||
k.vals.KV.(*kvMem).delErr = c.delErr
|
||||
runs := 0
|
||||
runEnded := func(e buildstep.StepEnded) error {
|
||||
runs++
|
||||
if _, err := k.body(d); err != nil {
|
||||
t.Errorf("the run's end is written after the input's delete (%d)", runs)
|
||||
}
|
||||
return c.runErr
|
||||
}
|
||||
raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"})
|
||||
m := &heardMsg{data: raw, subject: "step-ended"}
|
||||
heardStepEnded(k, runEnded, m, t.Logf)
|
||||
if m.naked != c.naked || m.acked == c.naked {
|
||||
t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked)
|
||||
}
|
||||
if m.naked && c.delErr != nil { // the redelivery
|
||||
m = &heardMsg{data: raw, subject: "step-ended"}
|
||||
heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf)
|
||||
}
|
||||
if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted {
|
||||
t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object
|
||||
@@ -346,3 +395,109 @@ func TestAssignerInputsSaysWhenTheKeyGoneObjectsCannotBeListed(t *testing.T) {
|
||||
t.Errorf("answer %+v: an unreadable store is not answered as no key-gone objects", m)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-465a: an input is kept once per content under its sha256 alone; every ask names the digest it uses, and the input
|
||||
// goes, with its object, at the step-ended of the last ask that names it, or a day after its last write; a request the cap still
|
||||
// refuses says the fill; an input of the old form (a key that is no digest) is ended with its object.
|
||||
func TestAnInputIsKeptOncePerContentAndEndsWithItsLastAsk(t *testing.T) {
|
||||
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
|
||||
objs := &objectsMem{objs: map[string][]byte{}}
|
||||
k := testKeeper(objs, func() time.Time { return now })
|
||||
body := bytes.Repeat([]byte{'n'}, 2<<20)
|
||||
var digests []string
|
||||
for _, ask := range []string{"ask-1", "ask-2", "ask-3"} {
|
||||
d, err := k.keep(ask, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digests = append(digests, d)
|
||||
}
|
||||
if digests[0] != digestOf(body) || digests[1] != digests[0] || digests[2] != digests[0] {
|
||||
t.Fatalf("digests = %v, want the sha256 of the bytes each time", digests)
|
||||
}
|
||||
if _, err := k.vals.KV.Get(digests[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(digests[0]); e == nil || len(objs.objs) != 1 {
|
||||
t.Fatalf("one key by the digest and one object wanted, got entry %v and %d objects", e, len(objs.objs))
|
||||
}
|
||||
for _, key := range keysOf(t, k) {
|
||||
if !strings.HasPrefix(key, inputsAskPrefix) && key != digests[0] {
|
||||
t.Errorf("a key %q beside the one input", key)
|
||||
}
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(inputsAskPrefix + "ask-2"); e == nil || !strings.Contains(string(e.Value), digests[0]) || !strings.Contains(string(e.Value), "20261117T100000Z") {
|
||||
t.Errorf("the ask does not name the digest it uses: %v", e)
|
||||
}
|
||||
for _, ask := range []string{"ask-1", "ask-2"} {
|
||||
if err := k.ended(ask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err != nil {
|
||||
t.Fatalf("the input went at the end of %s while ask-3 still names it: %v", ask, err)
|
||||
}
|
||||
}
|
||||
if err := k.ended("ask-3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 || len(keysOf(t, k)) != 0 {
|
||||
t.Errorf("the last step-ended left the input (%v), %d objects, keys %v", err, len(objs.objs), keysOf(t, k))
|
||||
}
|
||||
// a day after its last write, a renewed write counted
|
||||
if _, err := k.keep("ask-4", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(20 * time.Hour)
|
||||
if _, err := k.keep("ask-5", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(20 * time.Hour)
|
||||
if _, err := k.keep("ask-6", []byte("other")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err != nil {
|
||||
t.Errorf("an input written again 20 hours ago was swept: %v", err)
|
||||
}
|
||||
now = now.Add(25 * time.Hour)
|
||||
if _, err := k.keep("ask-7", []byte("another")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 {
|
||||
t.Errorf("an input a day after its last write is kept (%v), %d objects", err, len(objs.objs))
|
||||
}
|
||||
// an input of the old form goes a day after the time its key carries, with its object, never earlier
|
||||
old := "20261117T090000Z." + digestOf([]byte("old"))
|
||||
if err := k.vals.Put(old, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{'o'}, 2<<20))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent := now.Add(-time.Hour).Format(inputsTime) + "." + digestOf([]byte("recent"))
|
||||
if err := k.vals.Put(recent, "cmVjZW50"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := k.sweep(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(old); e != nil || len(objs.objs) != 0 {
|
||||
t.Errorf("the input of the old form is kept (%v) with %d objects", e, len(objs.objs))
|
||||
}
|
||||
if e, _ := k.vals.KV.Get(recent); e == nil {
|
||||
t.Error("an input of the old form under a day old was deleted early")
|
||||
}
|
||||
// a new content the cap refuses says the fill; the same content, already kept, needs no room
|
||||
full := newTestKeeperWithFill(objs, now, inputsCap-10)
|
||||
if _, err := full.keep("ask-8", []byte("fits")); err != nil {
|
||||
t.Fatalf("a content that fits is refused: %v", err)
|
||||
}
|
||||
if _, err := full.keep("ask-9", []byte("does not fit at all")); err == nil || !strings.Contains(err.Error(), "bytes kept") {
|
||||
t.Errorf("the refusal does not name the fill: %v", err)
|
||||
}
|
||||
if _, err := full.keep("ask-10", []byte("fits")); err != nil {
|
||||
t.Errorf("the content already kept is refused at the cap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestKeeperWithFill(objs *objectsMem, now time.Time, used int64) *inputKeeper {
|
||||
k := testKeeper(objs, func() time.Time { return now })
|
||||
k.used = func() (int64, error) { return used, nil }
|
||||
return k
|
||||
}
|
||||
|
||||
@@ -69,11 +69,13 @@ import (
|
||||
type bed struct {
|
||||
// seatListRead is closed once the assigner read the seat list (serveSeatList).
|
||||
seatListRead <-chan struct{}
|
||||
srv *server.Server
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
dir string
|
||||
buckets map[string]jetstream.KeyValue
|
||||
// settings are the files of settings a test sets, by the variable naming each (as the manifest's resources fill them).
|
||||
settings map[string]string
|
||||
srv *server.Server
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
dir string
|
||||
buckets map[string]jetstream.KeyValue
|
||||
// placed is, by node, the modules the bed's assignments names on it (place).
|
||||
placed map[string][]string
|
||||
// journal is what the programs the bed starts write in their journal.
|
||||
@@ -169,7 +171,7 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
continue
|
||||
}
|
||||
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
|
||||
if name == plc.PulsesBucket() {
|
||||
if name == plc.PulsesBucket() || name == buckets.CompositionRuns() {
|
||||
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, cfg)
|
||||
@@ -301,6 +303,13 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": settle}
|
||||
for name, value := range b.settings {
|
||||
file := filepath.Join(b.dir, name)
|
||||
if err := os.WriteFile(file, []byte(value), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env[name] = file
|
||||
}
|
||||
ctx, stop := context.WithCancel(b.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
|
||||
@@ -9,8 +9,10 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"io"
|
||||
"log"
|
||||
@@ -89,7 +91,7 @@ func ownBuckets() []string {
|
||||
return []string{buckets.Asks(), buckets.Differences(),
|
||||
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
|
||||
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
|
||||
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket()}
|
||||
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), buckets.CompositionRuns(), pinsBucket()}
|
||||
}
|
||||
|
||||
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
|
||||
@@ -133,11 +135,12 @@ var keptStates = []announce.State{
|
||||
{Name: copiesState, Readable: true},
|
||||
{Name: writtenState, Readable: true},
|
||||
{Name: buckets.WaitsState, Readable: true},
|
||||
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
|
||||
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
|
||||
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
|
||||
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
|
||||
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
|
||||
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
|
||||
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
|
||||
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
|
||||
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
|
||||
{Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c)
|
||||
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
|
||||
}
|
||||
|
||||
// kvSet is every bucket the assigner opened, by name.
|
||||
@@ -526,9 +529,34 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
// The build seat's step-ended of an ask deletes the input kept for it (design 47 G §354) and ends the composition run that waits
|
||||
// for it (A-240); the watchman's machine-away and machine-back say which machines a composition step does not compose (§64,
|
||||
// §389). Registered before the hub starts.
|
||||
ends, away := &runEnds{}, &awayNodes{}
|
||||
away := &awayNodes{}
|
||||
// The composition runs (design 47 G §718, §719), watched in their own state; a build seat's step-ended of a run's ask ends
|
||||
// the run in its key.
|
||||
var runs *compositionRuns
|
||||
if kv := kvs[buckets.CompositionRuns()]; kv != nil {
|
||||
set, err := runSettingsOf(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runs = newCompositionRuns(busRuns{ctx, js, kv}, set, time.Now, logger.Printf)
|
||||
if js != nil {
|
||||
runs.say = wantsPublisher(ctx, js, thisNode)
|
||||
}
|
||||
if bus != nil {
|
||||
runs.drop = dropOver(bus.CallKept)
|
||||
}
|
||||
// The build seats' ask-taken and step-progress begin a run's clock (§720, A-466q).
|
||||
if err := hearRunBounds(hub, runs, logger.Printf); err != nil {
|
||||
logger.Printf("the build seats' ask-taken and step-progress are not heard, so a composition run is ended by no bound: %v", err)
|
||||
}
|
||||
}
|
||||
if inputs != nil {
|
||||
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, ends, m, logger.Printf) }); err != nil {
|
||||
var runEnded func(buildstep.StepEnded) error
|
||||
if runs != nil && bus != nil {
|
||||
read := func(e buildstep.StepEnded) ([]byte, error) { return e.ReadOutput(buildstep.AskGo, bus.CallFile) }
|
||||
runEnded = func(e buildstep.StepEnded) error { return runs.heardEnd(e, read) }
|
||||
}
|
||||
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, runEnded, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -682,11 +710,40 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
|
||||
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
|
||||
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
|
||||
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
|
||||
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
|
||||
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
|
||||
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, ended: runs.stepEnded, now: time.Now, logf: logger.Printf,
|
||||
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
|
||||
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
|
||||
runs.moved.Store(&move)
|
||||
// The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up
|
||||
// once the values that stand are read (§719, §720).
|
||||
if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil {
|
||||
return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err)
|
||||
}
|
||||
// The runs' read and drop, the seat mesh-wants' verbs compositions and compositions-drop (§728, A-469f), answered here
|
||||
// where the runs are kept and the readers' ends said; the drop's hand act recorded through the seat mesh-hand-acts first.
|
||||
if e, ok := bus.(emitting); ok {
|
||||
record := func(q meshhandactsclient.Request) error {
|
||||
_, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q)
|
||||
return err
|
||||
}
|
||||
stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), compositionVerbs(runs, steps, record))
|
||||
if err != nil {
|
||||
logger.Printf("the verbs %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, err)
|
||||
} else {
|
||||
go func() { <-ctx.Done(); stopVerbs() }()
|
||||
}
|
||||
}
|
||||
// A superseded or cancelled pipeline's composition step leaves the runs it reads (§723, A-469n).
|
||||
if p.ends != nil {
|
||||
left := func(pipeline string) error {
|
||||
_, err := runs.leave(pipeline, steps.leave)
|
||||
return err
|
||||
}
|
||||
p.ends.left.Store(&left)
|
||||
}
|
||||
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
|
||||
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
|
||||
} else {
|
||||
|
||||
@@ -45,7 +45,9 @@ type passer struct {
|
||||
kvs kvSet
|
||||
w *world
|
||||
logf func(string, ...any)
|
||||
sdk string
|
||||
// unread is the set of modules whose running build's manifest served last could not read, said once per change of it.
|
||||
unread saidSet
|
||||
sdk string
|
||||
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(oc.Condition) error
|
||||
|
||||
@@ -1434,6 +1434,17 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
|
||||
t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want)
|
||||
}
|
||||
}
|
||||
// The modules whose running build's manifest is not read are said once per change of their set (G-23).
|
||||
var said []string
|
||||
p.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
lateOnly := assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"server": at(fp("f")), "late": at(fp("2"))}}}
|
||||
p.served(g, lateOnly)
|
||||
p.served(g, lateOnly)
|
||||
p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"late": at(fp("2")), "loop": at(fp("1"))}}})
|
||||
p.served(g, lateOnly)
|
||||
if len(said) != 2 || !strings.Contains(said[0], "[loop late]") && !strings.Contains(said[0], "[late loop]") || !strings.Contains(said[1], "[late]") {
|
||||
t.Errorf("the unread manifests were said %q, want once per change of the set", said)
|
||||
}
|
||||
// A-426a: a healthy build held (no catalog entry of it) or waiting for a yes serves what it serves, said once; one held unhealthy,
|
||||
// or whose running statement says it stopped, does not.
|
||||
held := at(fp("2"))
|
||||
|
||||
@@ -153,6 +153,9 @@ type pipelineEnds struct {
|
||||
firsts map[string]string
|
||||
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
|
||||
runEnded func(group string) error
|
||||
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
|
||||
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
|
||||
left atomic.Pointer[func(pipeline string) error]
|
||||
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
|
||||
onBranches func([]string)
|
||||
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
|
||||
@@ -285,6 +288,13 @@ func (e *pipelineEnds) branchOf(module, placed string) string {
|
||||
// §29, §64).
|
||||
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
|
||||
p, err := mprc.PipelineEnded(m.Data())
|
||||
if left := e.left.Load(); err == nil && left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
|
||||
if err := (*left)(p.Pipeline); err != nil {
|
||||
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
e.logf("%s is not taken: %v", m.Subject(), err)
|
||||
|
||||
@@ -423,6 +423,7 @@ func serveTools() error {
|
||||
tools = append(tools, wants.tools()...)
|
||||
keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}
|
||||
tools = append(tools, inputsRead(keeper), inputsDrop(keeper, stdio.Ask))
|
||||
tools = append(tools, compositionsTools(stdio.Ask)...) // the composition runs' read and drop, through their verbs (§728)
|
||||
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
|
||||
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
|
||||
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
|
||||
|
||||
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
@@ -53,12 +55,31 @@ func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
|
||||
}
|
||||
served, unread := graph.Served(servingHaves(n, p.countedServing), func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
|
||||
func(module string) []string { return g[module].Serves })
|
||||
if len(unread) > 0 {
|
||||
if p.unread.changed(unread) && len(unread) > 0 {
|
||||
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
|
||||
}
|
||||
return served
|
||||
}
|
||||
|
||||
// saidSet is the last set of names said, so a pass says a set again only when it changed.
|
||||
type saidSet struct {
|
||||
mu sync.Mutex
|
||||
last string
|
||||
}
|
||||
|
||||
// changed keeps names as the set said last and says whether it differs from the one before.
|
||||
func (s *saidSet) changed(names []string) bool {
|
||||
sorted := slices.Sorted(slices.Values(names))
|
||||
key := strings.Join(sorted, " ")
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if key == s.last {
|
||||
return false
|
||||
}
|
||||
s.last = key
|
||||
return true
|
||||
}
|
||||
|
||||
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
|
||||
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A have
|
||||
// waiting for a person's yes on a newer build counts too: it names the build that still runs and that build's health, and
|
||||
|
||||
@@ -29,6 +29,9 @@ const (
|
||||
PutWaitsState = "put-waits"
|
||||
// InputsState holds the inputs of the composition steps, `<time>.<digest>`, which the seat mesh-wants' verb input answers (design 47 G §354).
|
||||
InputsState = "inputs"
|
||||
// CompositionRunsState holds the composition runs, one key `<candidate key>.<machine>` per run and its bound keys
|
||||
// `bound.<run key>` and `untaken.<run key>` (design 47 G §719, §720, A-470).
|
||||
CompositionRunsState = "composition-runs"
|
||||
// PoliciesState is the state of the policies a person set for a module's upgrade.
|
||||
PoliciesState = "policies"
|
||||
)
|
||||
@@ -37,16 +40,17 @@ const (
|
||||
// constant of its name (design 47 G A-296, §428).
|
||||
func own(state string) string { return plc.Module() + "_" + state }
|
||||
|
||||
func Wants() string { return own(wants.StateWants) }
|
||||
func Differences() string { return own(wants.StateDifferences) }
|
||||
func Asks() string { return own("asks") }
|
||||
func Puts() string { return own("puts") }
|
||||
func Waits() string { return own(WaitsState) }
|
||||
func DeliveredAgain() string { return own(DeliveredAgainState) }
|
||||
func DeliveryWaits() string { return own(DeliveryWaitsState) }
|
||||
func PutWaits() string { return own(PutWaitsState) }
|
||||
func Inputs() string { return own(InputsState) }
|
||||
func Policies() string { return own(PoliciesState) }
|
||||
func Wants() string { return own(wants.StateWants) }
|
||||
func Differences() string { return own(wants.StateDifferences) }
|
||||
func Asks() string { return own("asks") }
|
||||
func Puts() string { return own("puts") }
|
||||
func Waits() string { return own(WaitsState) }
|
||||
func DeliveredAgain() string { return own(DeliveredAgainState) }
|
||||
func DeliveryWaits() string { return own(DeliveryWaitsState) }
|
||||
func PutWaits() string { return own(PutWaitsState) }
|
||||
func Inputs() string { return own(InputsState) }
|
||||
func CompositionRuns() string { return own(CompositionRunsState) }
|
||||
func Policies() string { return own(PoliciesState) }
|
||||
|
||||
// AsksHistory is the revisions the bucket of asks keeps per key.
|
||||
const AsksHistory = 5
|
||||
|
||||
@@ -31,7 +31,8 @@ func TestTheAssignersOwnBucketsTakeTheModuleItRunsAs(t *testing.T) {
|
||||
"Wants": {Wants(), "some-assigner_wants"}, "Differences": {Differences(), "some-assigner_differences"},
|
||||
"Asks": {Asks(), "some-assigner_asks"}, "Puts": {Puts(), "some-assigner_puts"}, "Policies": {Policies(), "some-assigner_policies"},
|
||||
"Waits": {Waits(), "some-assigner_waits"}, "Inputs": {Inputs(), "some-assigner_inputs"},
|
||||
"Assignments": {plc.AssignmentsBucket(), "some-assigner_assignments"}, "SeatHolders": {plc.SeatHoldersBucket(), "some-assigner_seat-holders"},
|
||||
"CompositionRuns": {CompositionRuns(), "some-assigner_composition-runs"},
|
||||
"Assignments": {plc.AssignmentsBucket(), "some-assigner_assignments"}, "SeatHolders": {plc.SeatHoldersBucket(), "some-assigner_seat-holders"},
|
||||
} {
|
||||
if c[0] != c[1] {
|
||||
t.Errorf("%s = %q, want %q", name, c[0], c[1])
|
||||
|
||||
@@ -70,7 +70,7 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
|
||||
"assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1,
|
||||
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "delivery-waits": 1, "delivered-again": 1, "inputs": 1, "pins": 5}; !reflect.DeepEqual(states, want) {
|
||||
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "delivery-waits": 1, "delivered-again": 1, "inputs": 1, "composition-runs": 1, "pins": 5}; !reflect.DeepEqual(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||
@@ -93,7 +93,7 @@ func TestManifest(t *testing.T) {
|
||||
// revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB.
|
||||
if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh ||
|
||||
!slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) ||
|
||||
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input"}) {
|
||||
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop"}) {
|
||||
t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims)
|
||||
}
|
||||
if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 {
|
||||
@@ -103,7 +103,7 @@ func TestManifest(t *testing.T) {
|
||||
// ADR 0621 §1: the first-view tools assignments, seat-holders, wants and asks (§717) and design 25 § 12.2's six read tools, of the
|
||||
// program's bundle, which runs in a runtime of its own on the assigner's account; ADR 0618 §3: and assigner_engine_reads,
|
||||
// which lists and asks again what each node's engine reads.
|
||||
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assignments", "seat-holders", "wants", "asks",
|
||||
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks",
|
||||
"assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies",
|
||||
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_pins", "assigner_unpin", "assigner_delivered_again"}) {
|
||||
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools)
|
||||
@@ -137,7 +137,7 @@ func TestManifest(t *testing.T) {
|
||||
// ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events;
|
||||
// issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears; §376, §389: its own version-delivered and
|
||||
// the watchman's machine-away and machine-back, read whole at its start.
|
||||
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.IndexerModule + ".take-in.taken", buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) {
|
||||
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.IndexerModule + ".take-in.taken", buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) {
|
||||
t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes)
|
||||
}
|
||||
if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
|
||||
@@ -147,7 +147,8 @@ func TestManifest(t *testing.T) {
|
||||
"seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations
|
||||
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
|
||||
"seat:node-backup.backed-up", "seat:node-backup.now",
|
||||
"seat:node-build-go.accept.check", // A-231: the composition run's ask on the Go build seat's queue check (accept.check.<asker>)
|
||||
"seat:node-build-go.accept.check", // A-231
|
||||
"seat:node-build-go.drop", // §720, A-469s: a composition run past its bound drops its ask: the composition run's ask on the Go build seat's queue check (accept.check.<asker>)
|
||||
"seat:node-build-go.output", // A-254: a composition run's answer kept as the build seat's object
|
||||
"seat:node-engine.compose-candidate", // §354, A-226: each machine's composition input
|
||||
"seat:node-engine.manifest-reads", // ADR 0618 §3
|
||||
@@ -167,13 +168,21 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
file, _ := os.ReadFile("module.json")
|
||||
if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) ||
|
||||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 3 {
|
||||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 8 ||
|
||||
raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" ||
|
||||
raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) {
|
||||
t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err)
|
||||
}
|
||||
for _, f := range []struct{ env, path, key string }{
|
||||
{"MESH_ASSIGNER_OUTSIDE_WAIT_FILE", "${dir:state}/outside-wait-minutes", "outside-wait-minutes"},
|
||||
{"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE", "${dir:state}/outside-settle-seconds", "outside-settle-seconds"},
|
||||
{"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE", "${dir:state}/ask-operator-for", "ask-operator-for"},
|
||||
// design 47 G §719, §720, §721, A-469x: the composition runs' settings
|
||||
{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE", "${dir:state}/composition-runs-cap-mib", "composition-runs-cap-mib"},
|
||||
{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE", "${dir:state}/composition-runs-kept", "composition-runs-kept"},
|
||||
{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE", "${dir:state}/composition-step-bound", "composition-step-bound"},
|
||||
{"MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE", "${dir:state}/run-untaken-bound", "run-untaken-bound"},
|
||||
{"MESH_ASSIGNER_BLAME_TRIES_FILE", "${dir:state}/blame-tries", "blame-tries"},
|
||||
} {
|
||||
if !strings.Contains(string(file), `"`+f.env+`": "`+f.path+`"`) || !strings.Contains(string(file), `"path": "`+f.path+`"`) ||
|
||||
!strings.Contains(string(file), `"content": "${config:`+f.key+`}"`) {
|
||||
|
||||
@@ -25,7 +25,9 @@
|
||||
"revision",
|
||||
"entry",
|
||||
"difference",
|
||||
"input"
|
||||
"input",
|
||||
"compositions",
|
||||
"compositions-drop"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -59,6 +61,8 @@
|
||||
"mesh-pipeline-runner.pipeline-ended",
|
||||
"mesh-pipeline-runner.group-ended",
|
||||
"node-build-go.step-ended",
|
||||
"node-build-go.ask-taken",
|
||||
"node-build-go.step-progress",
|
||||
"node-engine.manifest-reads-changed",
|
||||
"node-engine.running",
|
||||
"mesh-delivery.pinned",
|
||||
@@ -82,6 +86,7 @@
|
||||
"seat:node-backup.backed-up",
|
||||
"seat:node-backup.now",
|
||||
"seat:node-build-go.output",
|
||||
"seat:node-build-go.drop",
|
||||
"seat:node-engine.compose-candidate",
|
||||
"seat:node-engine.manifest-reads",
|
||||
"seat:operator-channel.asks",
|
||||
@@ -113,6 +118,8 @@
|
||||
"assigner_waits",
|
||||
"assigner_inputs",
|
||||
"assigner_inputs_drop",
|
||||
"assigner_compositions",
|
||||
"assigner_compositions_drop",
|
||||
"assignments",
|
||||
"seat-holders",
|
||||
"wants",
|
||||
@@ -158,6 +165,7 @@
|
||||
{ "name": "delivery-waits", "history": 1 },
|
||||
{ "name": "delivered-again", "history": 1 },
|
||||
{ "name": "inputs", "history": 1, "max-mib": 64 },
|
||||
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 },
|
||||
{ "name": "pins", "history": 5, "max-mib": 4 }
|
||||
],
|
||||
"config": {
|
||||
@@ -182,6 +190,40 @@
|
||||
"description": "the change kinds of a proposal the assigner asks the operator before it writes it, separated by commas: place, take-off, pin, unpin, hold, unhold, decide, config, add-node, remove-node, and stops for a verdict whose answers say a node would stop running a module (design 47 G A-425, A-425a). A proposal none of whose changes is of a kind named is applied at once when its verdict passes, said in the log and by assignments-changed. The default is take-off,remove-node,stops; empty asks for nothing.",
|
||||
"trusted": false,
|
||||
"default": "take-off,remove-node,stops"
|
||||
},
|
||||
"composition-runs-cap-mib": {
|
||||
"type": "number",
|
||||
"description": "the cap of the state composition-runs in MiB, the process refusing a new composition run at it with its steps ending could not run naming the fill (design 47 G section 719, A-469d); its maximum is the manifest's max-mib.",
|
||||
"minimum": 1,
|
||||
"maximum": 64,
|
||||
"trusted": false,
|
||||
"default": 16
|
||||
},
|
||||
"composition-runs-kept": {
|
||||
"type": "string",
|
||||
"description": "how long a composition run's key is kept after it ended, a Go duration such as 24h (design 47 G section 719, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "24h"
|
||||
},
|
||||
"composition-step-bound": {
|
||||
"type": "string",
|
||||
"description": "how long a taken composition run's ask may go without saying its progress before the run ends could not run, a Go duration such as 30m; written again at each step-progress (design 47 G section 720, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "30m"
|
||||
},
|
||||
"run-untaken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long a composition run's ask may wait for a build node to take it before the run ends could not run, a Go duration such as 24h (design 47 G section 720, A-466s, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "24h"
|
||||
},
|
||||
"blame-tries": {
|
||||
"type": "number",
|
||||
"description": "the most compositions a machine's composer makes to find the modules it blames, carried in the run's request as blame_tries (design 47 G section 721, A-466r).",
|
||||
"minimum": 1,
|
||||
"maximum": 1000,
|
||||
"trusted": false,
|
||||
"default": 20
|
||||
}
|
||||
},
|
||||
"objects": [
|
||||
@@ -231,6 +273,46 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:ask-operator-for}"
|
||||
},
|
||||
{
|
||||
"id": "composition-runs-cap-mib",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/composition-runs-cap-mib",
|
||||
"owner": "mesh-delivery-assigner",
|
||||
"mode": "0600",
|
||||
"content": "${config:composition-runs-cap-mib}"
|
||||
},
|
||||
{
|
||||
"id": "composition-runs-kept",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/composition-runs-kept",
|
||||
"owner": "mesh-delivery-assigner",
|
||||
"mode": "0600",
|
||||
"content": "${config:composition-runs-kept}"
|
||||
},
|
||||
{
|
||||
"id": "composition-step-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/composition-step-bound",
|
||||
"owner": "mesh-delivery-assigner",
|
||||
"mode": "0600",
|
||||
"content": "${config:composition-step-bound}"
|
||||
},
|
||||
{
|
||||
"id": "run-untaken-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/run-untaken-bound",
|
||||
"owner": "mesh-delivery-assigner",
|
||||
"mode": "0600",
|
||||
"content": "${config:run-untaken-bound}"
|
||||
},
|
||||
{
|
||||
"id": "blame-tries",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/blame-tries",
|
||||
"owner": "mesh-delivery-assigner",
|
||||
"mode": "0600",
|
||||
"content": "${config:blame-tries}"
|
||||
},
|
||||
{
|
||||
"id": "assigner",
|
||||
"type": "process",
|
||||
@@ -243,7 +325,12 @@
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_ASSIGNER_OUTSIDE_WAIT_FILE": "${dir:state}/outside-wait-minutes",
|
||||
"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": "${dir:state}/outside-settle-seconds",
|
||||
"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE": "${dir:state}/ask-operator-for"
|
||||
"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE": "${dir:state}/ask-operator-for",
|
||||
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "${dir:state}/composition-runs-cap-mib",
|
||||
"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "${dir:state}/composition-runs-kept",
|
||||
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "${dir:state}/composition-step-bound",
|
||||
"MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound",
|
||||
"MESH_ASSIGNER_BLAME_TRIES_FILE": "${dir:state}/blame-tries"
|
||||
},
|
||||
"user": "mesh-delivery-assigner",
|
||||
"health": {
|
||||
|
||||
@@ -15,18 +15,20 @@ import (
|
||||
const Seat = "mesh-pipeline-runner"
|
||||
|
||||
const (
|
||||
VerbRun = "run"
|
||||
VerbStart = "start"
|
||||
VerbShow = "show"
|
||||
VerbList = "list"
|
||||
VerbStands = "stands"
|
||||
VerbWhyCarried = "why-carried"
|
||||
VerbKept = "kept"
|
||||
VerbReview = "review"
|
||||
VerbRerun = "rerun"
|
||||
VerbAccept = "accept"
|
||||
VerbCancel = "cancel"
|
||||
VerbAskAgain = "ask-again"
|
||||
VerbRun = "run"
|
||||
VerbStart = "start"
|
||||
VerbShow = "show"
|
||||
VerbList = "list"
|
||||
VerbStands = "stands"
|
||||
VerbWhyCarried = "why-carried"
|
||||
VerbKept = "kept"
|
||||
VerbReview = "review"
|
||||
VerbRerun = "rerun"
|
||||
VerbAccept = "accept"
|
||||
VerbCancel = "cancel"
|
||||
VerbAskAgain = "ask-again"
|
||||
VerbTestRuns = "test-runs"
|
||||
VerbTestRunsDrop = "test-runs-drop"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -45,7 +47,7 @@ const (
|
||||
|
||||
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
|
||||
func Verbs() []string {
|
||||
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain}
|
||||
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain, VerbTestRuns, VerbTestRunsDrop}
|
||||
}
|
||||
|
||||
// Events is every event published on the seat's subject: a new copy each call.
|
||||
@@ -596,6 +598,26 @@ type RunStarted struct {
|
||||
V int64 `json:"v"`
|
||||
}
|
||||
|
||||
// SharedAskedBy is the step that asked the run, for a test step reading another's run.
|
||||
type SharedAskedBy struct {
|
||||
// its artifact, <module>:<artifact>
|
||||
Artifact string `json:"artifact"`
|
||||
// its pipeline's key
|
||||
Pipeline string `json:"pipeline"`
|
||||
}
|
||||
|
||||
// Shared is a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself.
|
||||
type Shared struct {
|
||||
// the id of the run's ask of the build seat
|
||||
Ask string `json:"ask"`
|
||||
// the step that asked the run, for a test step reading another's run
|
||||
AskedBy *SharedAskedBy `json:"asked-by,omitempty"`
|
||||
// the machine composed, for a composition run
|
||||
Machine string `json:"machine,omitempty"`
|
||||
// the run's key: in the assigner's composition-runs, or the runner's test-runs
|
||||
Run string `json:"run"`
|
||||
}
|
||||
|
||||
// StepEvent is a step's start or end: the step, its status, its outcome, its fingerprint, its test counts, what a carried result was carried from, and for a rerun that it is one and its proved caller; a bypassed step too.
|
||||
type StepEvent struct {
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
@@ -645,7 +667,9 @@ type StepResult struct {
|
||||
// the step was asked again by the runner's rerun
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
// the seat that serves the step, whose holder alone may end it (ADR 0666 section 21)
|
||||
Seat string `json:"seat,omitempty"`
|
||||
Seat string `json:"seat,omitempty"`
|
||||
// the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t)
|
||||
Shared []Shared `json:"shared,omitempty"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
// the step's status in the module's policy
|
||||
Status string `json:"status"`
|
||||
@@ -698,6 +722,56 @@ type TestCounts struct {
|
||||
Unit int64 `json:"unit,omitempty"`
|
||||
}
|
||||
|
||||
// TestRun is one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u).
|
||||
type TestRun struct {
|
||||
// the id of the build seat's ask
|
||||
Ask string `json:"ask,omitempty"`
|
||||
// the share digest (design 47 G §724)
|
||||
Digest string `json:"digest"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
// why the packages are not named: the run's output is gone
|
||||
Gone string `json:"gone,omitempty"`
|
||||
// the key, <candidate key>.<level>.<digest>, each part escaped as a key's name
|
||||
Key string `json:"key"`
|
||||
Level string `json:"level"`
|
||||
// the node whose holder took the ask
|
||||
Node string `json:"node,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
// the run's output object, by its name in test-runs-objects
|
||||
Output string `json:"output,omitempty"`
|
||||
// the packages it ran, read from its output object
|
||||
Packages []TestRunPackage `json:"packages,omitempty"`
|
||||
// the steps reading the run
|
||||
Readers []TestRunReader `json:"readers,omitempty"`
|
||||
// the outcome's words
|
||||
Said string `json:"said,omitempty"`
|
||||
// the build seat asked
|
||||
Seat string `json:"seat,omitempty"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
Tests *TestCounts `json:"tests,omitempty"`
|
||||
}
|
||||
|
||||
// The values of TestRun's level.
|
||||
const (
|
||||
TestRunLevelLint = "lint"
|
||||
TestRunLevelUnit = "unit"
|
||||
TestRunLevelIntegration = "integration"
|
||||
)
|
||||
|
||||
// TestRunPackage is one package a test run ran, with its digest (design 47 G §724).
|
||||
type TestRunPackage struct {
|
||||
Digest string `json:"digest"`
|
||||
Package string `json:"package"`
|
||||
}
|
||||
|
||||
// TestRunReader is a test step reading a test run (design 47 G §725).
|
||||
type TestRunReader struct {
|
||||
// its artifact, <module>:<artifact>
|
||||
Artifact string `json:"artifact"`
|
||||
// its pipeline's key
|
||||
Pipeline string `json:"pipeline"`
|
||||
}
|
||||
|
||||
// ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe.
|
||||
type ToolchainUsed struct {
|
||||
Language string `json:"language"`
|
||||
@@ -708,6 +782,8 @@ type ToolchainUsed struct {
|
||||
// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283).
|
||||
type Wait struct {
|
||||
Artifact string `json:"artifact,omitempty"`
|
||||
// the step's own ask, where it has one (design 47 G §731)
|
||||
Ask string `json:"ask,omitempty"`
|
||||
// the watched branch, or the branch of a run on request
|
||||
Branch string `json:"branch"`
|
||||
// the commit the module's entry was last changed at
|
||||
@@ -726,8 +802,10 @@ type Wait struct {
|
||||
// the catalog graph revision of the branch the pipeline runs for
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// the run on request's id, for a pipeline of a run; such a pipeline delivers nothing
|
||||
Run string `json:"run,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Run string `json:"run,omitempty"`
|
||||
// the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)
|
||||
Shared []Shared `json:"shared,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
|
||||
Source int64 `json:"source,omitempty"`
|
||||
// a pipeline step a wait inside a pipeline is in
|
||||
@@ -977,6 +1055,44 @@ type AskAgainAnswer struct {
|
||||
Result StepResult `json:"result"`
|
||||
}
|
||||
|
||||
// TestRunsInput is what test-runs takes.
|
||||
type TestRunsInput struct {
|
||||
// the candidate's key
|
||||
Candidate string `json:"candidate,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Level string `json:"level,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
}
|
||||
|
||||
// The values of TestRunsInput's level.
|
||||
const (
|
||||
TestRunsInputLevelLint = "lint"
|
||||
TestRunsInputLevelUnit = "unit"
|
||||
TestRunsInputLevelIntegration = "integration"
|
||||
)
|
||||
|
||||
// TestRunsAnswer is what test-runs answers.
|
||||
type TestRunsAnswer struct {
|
||||
Fill Fill `json:"fill"`
|
||||
Runs []TestRun `json:"runs"`
|
||||
}
|
||||
|
||||
// TestRunsDropInput is what test-runs-drop takes.
|
||||
type TestRunsDropInput struct {
|
||||
// the run's key in test-runs
|
||||
Key string `json:"key"`
|
||||
// why, written with the caller the runtime proves in the hand-act log
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// TestRunsDropAnswer is what test-runs-drop answers.
|
||||
type TestRunsDropAnswer struct {
|
||||
// the readers it ended
|
||||
Ended []TestRunReader `json:"ended,omitempty"`
|
||||
// the entry deleted
|
||||
Run TestRun `json:"run"`
|
||||
}
|
||||
|
||||
// PipelineStartedEvent is the body of the event pipeline-started.
|
||||
type PipelineStartedEvent = PipelineStarted
|
||||
|
||||
|
||||
@@ -37,6 +37,12 @@ func TestFixtures(t *testing.T) {
|
||||
decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.wait-held\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "the wait ended answered", subject: "mesh.seat.mesh-pipeline-runner.event.wait-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"how\":\"answered\",\"module\":\"nats\",\"on\":\"person\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"since\":\"2026-11-10T10:10:00Z\",\"source\":3589,\"step\":\"approval\",\"v\":1,\"who\":\"the operator\"}", wire: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"how\":\"answered\",\"module\":\"nats\",\"on\":\"person\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"since\":\"2026-11-10T10:10:00Z\",\"source\":3589,\"step\":\"approval\",\"v\":1,\"who\":\"the operator\"}",
|
||||
decode: func(b []byte) (any, error) { var v WaitEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.wait-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "C1/AC2 a test step started reading a test run, naming its run and ask", subject: "mesh.seat.mesh-pipeline-runner.event.step-started.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:00:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"outcome\":\"running\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\"},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:00:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"outcome\":\"running\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\"},\"revision\":12,\"source\":3589,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v StepStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-started\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "C1/AC2 a test step ended with the run another step asked", subject: "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"asked-by\":{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"asked-by\":{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},\"revision\":12,\"source\":3589,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "C1 a composition step ended with mesh-wants' runs as they came", subject: "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"fingerprint\":\"fp-0123\",\"node\":\"laptop\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}],\"status\":\"required\",\"step\":\"composition\"},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"fingerprint\":\"fp-0123\",\"node\":\"laptop\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}],\"status\":\"required\",\"step\":\"composition\"},\"revision\":12,\"source\":3589,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
|
||||
{name: "a pipeline kept", subject: "", given: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.11.nats\",\"fingerprint\":\"fp-0123\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.11.nats\",\"fingerprint\":\"fp-0123\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}",
|
||||
decode: func(b []byte) (any, error) {
|
||||
var v PipelinesByBranchRevisionModuleValue
|
||||
@@ -67,6 +73,12 @@ func TestFixtures(t *testing.T) {
|
||||
decode: func(b []byte) (any, error) { var v StandsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "kept answers by digest", subject: "", given: "{\"fill\":{\"bytes\":300,\"cap\":134217728,\"store\":\"kept\"},\"kept\":[{\"digest\":\"sha256:aa\",\"level\":\"unit\",\"package\":\"git.novox.be/novox/nats\",\"pipeline\":\"main.12.nats\"}]}", wire: "{\"fill\":{\"bytes\":300,\"cap\":134217728,\"store\":\"kept\"},\"kept\":[{\"digest\":\"sha256:aa\",\"level\":\"unit\",\"package\":\"git.novox.be/novox/nats\",\"pipeline\":\"main.12.nats\"}]}",
|
||||
decode: func(b []byte) (any, error) { var v KeptAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "C1/AC3 stands answers a waiting test step with its ask and the run it waits on", subject: "", given: "{\"stands\":[{\"artifact\":\"nats:tools\",\"ask\":\"t1a2\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"on\":\"build node\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"since\":\"2026-11-17T10:00:00Z\",\"source\":3589,\"step\":\"unit\",\"v\":1}]}", wire: "{\"stands\":[{\"artifact\":\"nats:tools\",\"ask\":\"t1a2\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"on\":\"build node\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"since\":\"2026-11-17T10:00:00Z\",\"source\":3589,\"step\":\"unit\",\"v\":1}]}",
|
||||
decode: func(b []byte) (any, error) { var v StandsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "C1 test-runs answers each run with its packages, readers and the fill", subject: "", given: "{\"fill\":{\"bytes\":2048,\"cap\":16777216,\"store\":\"test-runs\"},\"runs\":[{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}},{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"gone\":\"the run's output is gone\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}]}", wire: "{\"fill\":{\"bytes\":2048,\"cap\":16777216,\"store\":\"test-runs\"},\"runs\":[{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}},{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"gone\":\"the run's output is gone\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}]}",
|
||||
decode: func(b []byte) (any, error) { var v TestRunsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "C1 test-runs-drop answers the run it deleted and the readers it ended", subject: "", given: "{\"ended\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"run\":{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}}", wire: "{\"ended\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"run\":{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}}",
|
||||
decode: func(b []byte) (any, error) { var v TestRunsDropAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
}
|
||||
for _, c := range cases {
|
||||
v, err := c.decode([]byte(c.given))
|
||||
|
||||
@@ -15,12 +15,14 @@ import (
|
||||
const Seat = "mesh-wants"
|
||||
|
||||
const (
|
||||
VerbUpgrade = "upgrade"
|
||||
VerbBus = "bus"
|
||||
VerbRevision = "revision"
|
||||
VerbEntry = "entry"
|
||||
VerbDifference = "difference"
|
||||
VerbInput = "input"
|
||||
VerbUpgrade = "upgrade"
|
||||
VerbBus = "bus"
|
||||
VerbRevision = "revision"
|
||||
VerbEntry = "entry"
|
||||
VerbDifference = "difference"
|
||||
VerbInput = "input"
|
||||
VerbCompositions = "compositions"
|
||||
VerbCompositionsDrop = "compositions-drop"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -44,7 +46,7 @@ const (
|
||||
|
||||
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
|
||||
func Verbs() []string {
|
||||
return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput}
|
||||
return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop}
|
||||
}
|
||||
|
||||
// Events is every event published on the seat's subject: a new copy each call.
|
||||
@@ -125,7 +127,6 @@ const (
|
||||
StateDifferences = "differences"
|
||||
StateCompositions = "compositions"
|
||||
StateAssignmentRevisions = "assignment-revisions"
|
||||
StateInputs = "inputs"
|
||||
)
|
||||
|
||||
// WantsByWantNodeModuleKey is the key of one value of the state wants under the form "want.<node>.<module>".
|
||||
@@ -741,42 +742,6 @@ func DecodeAssignmentRevisionsByEntryDigestValue(raw []byte) (AssignmentRevision
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// InputsByTimeDigestKey is the key of one value of the state inputs under the form "<time>.<digest>".
|
||||
func InputsByTimeDigestKey(time, digest string) string { return time + "." + digest }
|
||||
|
||||
// ParseInputsByTimeDigestKey reads the parts of a key of the form "<time>.<digest>", in the order of the form; the last result is false for a key of another form.
|
||||
func ParseInputsByTimeDigestKey(key string) (string, string, bool) {
|
||||
rest := key
|
||||
v0, after0, found0 := strings.Cut(rest, ".")
|
||||
if !found0 || v0 == "" {
|
||||
return "", "", false
|
||||
}
|
||||
rest = after0
|
||||
if rest == "" {
|
||||
return "", "", false
|
||||
}
|
||||
v2 := rest
|
||||
rest = ""
|
||||
if rest != "" {
|
||||
return "", "", false
|
||||
}
|
||||
return v0, v2, true
|
||||
}
|
||||
|
||||
// DecodeInputsByTimeDigestValue reads one value of the state inputs under the form "<time>.<digest>", refusing a field InputsByTimeDigestValue does not know and any text after the value.
|
||||
func DecodeInputsByTimeDigestValue(raw []byte) (InputsByTimeDigestValue, error) {
|
||||
var v InputsByTimeDigestValue
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return v, err
|
||||
}
|
||||
if dec.More() {
|
||||
return v, errors.New("text after the value")
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// Ask is an ask the assigner puts to a person: the router's ask whole, as the assigner composes it, and how many times it was written at its key.
|
||||
type Ask struct {
|
||||
// the seat operator-channel's ask, whole
|
||||
@@ -867,6 +832,79 @@ type Change struct {
|
||||
To string `json:"to,omitempty"`
|
||||
}
|
||||
|
||||
// CompositionReader is a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G §719, A-468).
|
||||
type CompositionReader struct {
|
||||
// the watched branch, or the branch of a run on request
|
||||
Branch string `json:"branch"`
|
||||
// for a rerun, the caller the runtime proved
|
||||
By string `json:"by,omitempty"`
|
||||
// the commit the step runs at
|
||||
Commit string `json:"commit"`
|
||||
// when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m)
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
// the module's build-input fingerprint at the commit
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
// the step's ask's id
|
||||
ID string `json:"id"`
|
||||
Module string `json:"module"`
|
||||
// the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is
|
||||
Pipeline string `json:"pipeline"`
|
||||
// the step is asked again by the runner's rerun
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
// the catalog graph revision of the branch the pipeline runs for
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// the run on request's id: its composition writes no want (ADR 0666 section 18)
|
||||
Run string `json:"run,omitempty"`
|
||||
// the source revision of the watched branch; absent for a run on request
|
||||
Source int64 `json:"source,omitempty"`
|
||||
Step string `json:"step"`
|
||||
}
|
||||
|
||||
// The values of CompositionReader's step.
|
||||
const (
|
||||
CompositionReaderStepComposition = "composition"
|
||||
)
|
||||
|
||||
// CompositionRunProgram is the composer program, chosen once when the run started.
|
||||
type CompositionRunProgram struct {
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Directory string `json:"directory,omitempty"`
|
||||
// the node-engine fingerprint it was chosen from
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
}
|
||||
|
||||
// CompositionRun is one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G §719, §728; A-469u).
|
||||
type CompositionRun struct {
|
||||
// the id of the build seat's run ask
|
||||
Ask string `json:"ask,omitempty"`
|
||||
// the candidate's key: <branch>.<revision>, or run-<id> for a run on request
|
||||
Candidate string `json:"candidate"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
// the digest of the kept request
|
||||
Input string `json:"input,omitempty"`
|
||||
// the key, <candidate key>.<machine>, each part escaped as a key's name
|
||||
Key string `json:"key"`
|
||||
Machine string `json:"machine"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
// the composer program, chosen once when the run started
|
||||
Program *CompositionRunProgram `json:"program,omitempty"`
|
||||
// the composition steps reading the run
|
||||
Readers []CompositionReader `json:"readers,omitempty"`
|
||||
// the outcome's words
|
||||
Said string `json:"said,omitempty"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
// that machine's composer verdict (design 47 G §721)
|
||||
Verdict map[string]any `json:"verdict,omitempty"`
|
||||
}
|
||||
|
||||
// The values of CompositionRun's outcome.
|
||||
const (
|
||||
CompositionRunOutcomePassed = "passed"
|
||||
CompositionRunOutcomeFailed = "failed"
|
||||
CompositionRunOutcomeCouldNotRun = "could not run"
|
||||
)
|
||||
|
||||
// DifferenceRefusedItem is a type of the contract.
|
||||
type DifferenceRefusedItem struct {
|
||||
Module string `json:"module"`
|
||||
@@ -944,6 +982,26 @@ type NodeEntryRef struct {
|
||||
Node string `json:"node"`
|
||||
}
|
||||
|
||||
// SharedAskedBy is the step that asked the run, for a test step reading another's run.
|
||||
type SharedAskedBy struct {
|
||||
// its artifact, <module>:<artifact>
|
||||
Artifact string `json:"artifact"`
|
||||
// its pipeline's key
|
||||
Pipeline string `json:"pipeline"`
|
||||
}
|
||||
|
||||
// Shared is a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself.
|
||||
type Shared struct {
|
||||
// the id of the run's ask of the build seat
|
||||
Ask string `json:"ask"`
|
||||
// the step that asked the run, for a test step reading another's run
|
||||
AskedBy *SharedAskedBy `json:"asked-by,omitempty"`
|
||||
// the machine composed, for a composition run
|
||||
Machine string `json:"machine,omitempty"`
|
||||
// the run's key: in the assigner's composition-runs, or the runner's test-runs
|
||||
Run string `json:"run"`
|
||||
}
|
||||
|
||||
// StepEnded is the composition step's end, said by the assigner on the node it runs on (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at.
|
||||
type StepEnded struct {
|
||||
// for a rerun, the caller the runtime proved
|
||||
@@ -961,7 +1019,9 @@ type StepEnded struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
// why it failed or could not run: the two modules owning one path, the part not heard
|
||||
Said string `json:"said,omitempty"`
|
||||
Said string `json:"said,omitempty"`
|
||||
// per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G §723, §729)
|
||||
Shared []Shared `json:"shared,omitempty"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
Step string `json:"step"`
|
||||
// the layout version
|
||||
@@ -1151,6 +1211,36 @@ type InputAnswer struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// CompositionsInput is what compositions takes.
|
||||
type CompositionsInput struct {
|
||||
// the candidate's key
|
||||
Candidate string `json:"candidate,omitempty"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
}
|
||||
|
||||
// CompositionsAnswer is what compositions answers.
|
||||
type CompositionsAnswer struct {
|
||||
Fill Fill `json:"fill"`
|
||||
Runs []CompositionRun `json:"runs"`
|
||||
}
|
||||
|
||||
// CompositionsDropInput is what compositions-drop takes.
|
||||
type CompositionsDropInput struct {
|
||||
// the candidate's key
|
||||
Candidate string `json:"candidate"`
|
||||
Machine string `json:"machine"`
|
||||
// why, written with the caller the runtime proves in the hand-act log
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// CompositionsDropAnswer is what compositions-drop answers.
|
||||
type CompositionsDropAnswer struct {
|
||||
// the readers it ended
|
||||
Ended []CompositionReader `json:"ended,omitempty"`
|
||||
// the entry deleted
|
||||
Run CompositionRun `json:"run"`
|
||||
}
|
||||
|
||||
// StepAccept is the body of step.
|
||||
type StepAccept = StepRun
|
||||
|
||||
@@ -1234,6 +1324,8 @@ type StepProgressEvent struct {
|
||||
Asker string `json:"asker"`
|
||||
At time.Time `json:"at"`
|
||||
Phase string `json:"phase"`
|
||||
// the composition runs the step waits on, by machine, each with its ask (design 47 G §723, §729)
|
||||
Shared []Shared `json:"shared,omitempty"`
|
||||
}
|
||||
|
||||
// The values of StepProgressEvent's phase.
|
||||
@@ -1355,6 +1447,3 @@ type AssignmentRevisionsByCandidateGroupHeadsValue = AssignmentRevision
|
||||
|
||||
// AssignmentRevisionsByEntryDigestValue is one value of the state assignment-revisions under the key entry.<digest>.
|
||||
type AssignmentRevisionsByEntryDigestValue = AssignmentEntry
|
||||
|
||||
// InputsByTimeDigestValue is one value of the state inputs under the key <time>.<digest>.
|
||||
type InputsByTimeDigestValue = string
|
||||
|
||||
@@ -59,6 +59,10 @@ func TestFixtures(t *testing.T) {
|
||||
decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-held$")},
|
||||
{name: "an outside wait ended by the operator's Go on", subject: "mesh.seat.mesh-wants.event.wait-ended", given: "{\"fingerprint\":\"src2:aaaa\",\"how\":\"go-on\",\"module\":\"nats\",\"node\":\"anchor\",\"on\":\"silent:home-server\",\"owner\":\"mesh-delivery-assigner\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"outside\"}", wire: "{\"fingerprint\":\"src2:aaaa\",\"how\":\"go-on\",\"module\":\"nats\",\"node\":\"anchor\",\"on\":\"silent:home-server\",\"owner\":\"mesh-delivery-assigner\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"outside\"}",
|
||||
decode: func(b []byte) (any, error) { var v WaitEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-ended$")},
|
||||
{name: "C1/AC1 a reading composition step ended, naming per machine the run it read and its ask", subject: "mesh.seat.mesh-wants.event.step-ended.laptop", given: "{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"node\":\"laptop\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"},{\"ask\":\"r4d5e6\",\"machine\":\"home-server\",\"run\":\"main.12.home-server\"}],\"step\":\"composition\",\"v\":1,\"warnings\":[\"anchor holds xclip: xclip and demo both own /etc/x\"]}", wire: "{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"node\":\"laptop\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"},{\"ask\":\"r4d5e6\",\"machine\":\"home-server\",\"run\":\"main.12.home-server\"}],\"step\":\"composition\",\"v\":1,\"warnings\":[\"anchor holds xclip: xclip and demo both own /etc/x\"]}",
|
||||
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-ended\\.[^.*> ]+$")},
|
||||
{name: "C1/AC1 a composition step's progress, naming the runs it waits on", subject: "mesh.seat.mesh-wants.event.step-progress.anchor", given: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-17T10:01:00Z\",\"phase\":\"run\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}]}", wire: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-17T10:01:00Z\",\"phase\":\"run\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}]}",
|
||||
decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-progress\\.[^.*> ]+$")},
|
||||
{name: "a want", subject: "", given: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", wire: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}",
|
||||
decode: func(b []byte) (any, error) {
|
||||
var v WantsByWantNodeModuleValue
|
||||
@@ -141,6 +145,12 @@ func TestFixtures(t *testing.T) {
|
||||
decode: func(b []byte) (any, error) { var v InputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "input answers how many chunks carry the exact bytes", subject: "", given: "{\"chunks\":3,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", wire: "{\"chunks\":3,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}",
|
||||
decode: func(b []byte) (any, error) { var v InputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "C1 compositions answers each run with its readers, verdict and the fill", subject: "", given: "{\"fill\":{\"bytes\":4096,\"cap\":16777216,\"store\":\"composition-runs\"},\"runs\":[{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}]}", wire: "{\"fill\":{\"bytes\":4096,\"cap\":16777216,\"store\":\"composition-runs\"},\"runs\":[{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}]}",
|
||||
decode: func(b []byte) (any, error) { var v CompositionsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "C1 compositions-drop takes a candidate, a machine and why", subject: "", given: "{\"candidate\":\"main.12\",\"machine\":\"anchor\",\"why\":\"its ask never came back\"}", wire: "{\"candidate\":\"main.12\",\"machine\":\"anchor\",\"why\":\"its ask never came back\"}",
|
||||
decode: func(b []byte) (any, error) { var v CompositionsDropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "C1 compositions-drop answers the run it deleted and the readers it ended", subject: "", given: "{\"ended\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"run\":{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}}", wire: "{\"ended\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"run\":{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}}",
|
||||
decode: func(b []byte) (any, error) { var v CompositionsDropAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
}
|
||||
for _, c := range cases {
|
||||
v, err := c.decode([]byte(c.given))
|
||||
|
||||
@@ -313,5 +313,104 @@
|
||||
"how": "answered",
|
||||
"fingerprint": "fp-0123"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1/AC2 a test step started reading a test run, naming its run and ask",
|
||||
"of": "event:step-started",
|
||||
"subject": "mesh.seat.mesh-pipeline-runner.event.step-started.nats.main.0123456789abcdef0123456789abcdef01234567",
|
||||
"given": {
|
||||
"pipeline": "main.12.nats",
|
||||
"module": "nats",
|
||||
"branch": "main",
|
||||
"revision": 12,
|
||||
"source": 3589,
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"v": 1,
|
||||
"fingerprint": "fp-0123",
|
||||
"result": {
|
||||
"step": "unit",
|
||||
"artifact": "nats:tools",
|
||||
"status": "required",
|
||||
"outcome": "running",
|
||||
"fingerprint": "fp-0123",
|
||||
"seat": "node-build-go",
|
||||
"shared": [
|
||||
{
|
||||
"run": "main.12.unit.d1",
|
||||
"ask": "t1a2"
|
||||
}
|
||||
]
|
||||
},
|
||||
"at": "2026-11-17T10:00:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1/AC2 a test step ended with the run another step asked",
|
||||
"of": "event:step-ended",
|
||||
"subject": "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567",
|
||||
"given": {
|
||||
"pipeline": "main.12.nats",
|
||||
"module": "nats",
|
||||
"branch": "main",
|
||||
"revision": 12,
|
||||
"source": 3589,
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"v": 1,
|
||||
"fingerprint": "fp-0123",
|
||||
"result": {
|
||||
"step": "unit",
|
||||
"artifact": "nats:tools",
|
||||
"status": "required",
|
||||
"outcome": "failed",
|
||||
"fingerprint": "fp-0123",
|
||||
"seat": "node-build-go",
|
||||
"node": "anchor",
|
||||
"tests": {
|
||||
"unit": 42
|
||||
},
|
||||
"shared": [
|
||||
{
|
||||
"run": "main.12.unit.d1",
|
||||
"ask": "t1a2",
|
||||
"asked-by": {
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:program"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"at": "2026-11-17T10:05:00Z"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1 a composition step ended with mesh-wants' runs as they came",
|
||||
"of": "event:step-ended",
|
||||
"subject": "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567",
|
||||
"given": {
|
||||
"pipeline": "main.12.nats",
|
||||
"module": "nats",
|
||||
"branch": "main",
|
||||
"revision": 12,
|
||||
"source": 3589,
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"v": 1,
|
||||
"fingerprint": "fp-0123",
|
||||
"result": {
|
||||
"step": "composition",
|
||||
"status": "required",
|
||||
"outcome": "passed",
|
||||
"fingerprint": "fp-0123",
|
||||
"seat": "mesh-wants",
|
||||
"node": "laptop",
|
||||
"shared": [
|
||||
{
|
||||
"machine": "anchor",
|
||||
"run": "main.12.anchor",
|
||||
"ask": "r1a2b3"
|
||||
}
|
||||
]
|
||||
},
|
||||
"at": "2026-11-17T10:05:00Z"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -143,5 +143,154 @@
|
||||
"cap": 134217728
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1/AC3 stands answers a waiting test step with its ask and the run it waits on",
|
||||
"of": "verb:stands.answer",
|
||||
"given": {
|
||||
"stands": [
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"module": "nats",
|
||||
"branch": "main",
|
||||
"revision": 12,
|
||||
"source": 3589,
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"v": 1,
|
||||
"fingerprint": "fp-0123",
|
||||
"step": "unit",
|
||||
"artifact": "nats:tools",
|
||||
"on": "build node",
|
||||
"since": "2026-11-17T10:00:00Z",
|
||||
"ask": "t1a2",
|
||||
"shared": [
|
||||
{
|
||||
"run": "main.12.unit.d1",
|
||||
"ask": "t1a2"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1 test-runs answers each run with its packages, readers and the fill",
|
||||
"of": "verb:test-runs.answer",
|
||||
"given": {
|
||||
"runs": [
|
||||
{
|
||||
"key": "main.12.unit.d1",
|
||||
"level": "unit",
|
||||
"digest": "d1",
|
||||
"seat": "node-build-go",
|
||||
"ask": "t1a2",
|
||||
"node": "anchor",
|
||||
"started": "2026-11-17T10:00:00Z",
|
||||
"readers": [
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:program"
|
||||
},
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:tools"
|
||||
}
|
||||
],
|
||||
"ended": "2026-11-17T10:05:00Z",
|
||||
"outcome": "failed",
|
||||
"said": "1 test failed",
|
||||
"tests": {
|
||||
"unit": 42
|
||||
},
|
||||
"output": "t1a2",
|
||||
"packages": [
|
||||
{
|
||||
"package": "git.novox.be/novox/nats/internal/x",
|
||||
"digest": "sha256:aa"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "main.12.unit.d1",
|
||||
"level": "unit",
|
||||
"digest": "d1",
|
||||
"seat": "node-build-go",
|
||||
"ask": "t1a2",
|
||||
"node": "anchor",
|
||||
"started": "2026-11-17T10:00:00Z",
|
||||
"readers": [
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:program"
|
||||
},
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:tools"
|
||||
}
|
||||
],
|
||||
"ended": "2026-11-17T10:05:00Z",
|
||||
"outcome": "failed",
|
||||
"said": "1 test failed",
|
||||
"tests": {
|
||||
"unit": 42
|
||||
},
|
||||
"output": "t1a2",
|
||||
"gone": "the run's output is gone"
|
||||
}
|
||||
],
|
||||
"fill": {
|
||||
"store": "test-runs",
|
||||
"bytes": 2048,
|
||||
"cap": 16777216
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1 test-runs-drop answers the run it deleted and the readers it ended",
|
||||
"of": "verb:test-runs-drop.answer",
|
||||
"given": {
|
||||
"run": {
|
||||
"key": "main.12.unit.d1",
|
||||
"level": "unit",
|
||||
"digest": "d1",
|
||||
"seat": "node-build-go",
|
||||
"ask": "t1a2",
|
||||
"node": "anchor",
|
||||
"started": "2026-11-17T10:00:00Z",
|
||||
"readers": [
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:program"
|
||||
},
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:tools"
|
||||
}
|
||||
],
|
||||
"ended": "2026-11-17T10:05:00Z",
|
||||
"outcome": "failed",
|
||||
"said": "1 test failed",
|
||||
"tests": {
|
||||
"unit": 42
|
||||
},
|
||||
"output": "t1a2",
|
||||
"packages": [
|
||||
{
|
||||
"package": "git.novox.be/novox/nats/internal/x",
|
||||
"digest": "sha256:aa"
|
||||
}
|
||||
]
|
||||
},
|
||||
"ended": [
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:program"
|
||||
},
|
||||
{
|
||||
"pipeline": "main.12.nats",
|
||||
"artifact": "nats:tools"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -495,6 +495,93 @@
|
||||
]
|
||||
},
|
||||
"class": "act"
|
||||
},
|
||||
{
|
||||
"name": "test-runs",
|
||||
"class": "read",
|
||||
"description": "Each test run the runner keeps (design 47 G §728): its packages read from its output object, ask, node, readers, outcome and counts, and the state's fill against its cap; with candidate, level, digest or module, only those.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"candidate": {
|
||||
"type": "string",
|
||||
"description": "the candidate's key"
|
||||
},
|
||||
"level": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"lint",
|
||||
"unit",
|
||||
"integration"
|
||||
]
|
||||
},
|
||||
"digest": {
|
||||
"type": "string"
|
||||
},
|
||||
"module": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"runs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/TestRun"
|
||||
}
|
||||
},
|
||||
"fill": {
|
||||
"$ref": "#/defs/Fill"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"runs",
|
||||
"fill"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "test-runs-drop",
|
||||
"class": "act",
|
||||
"description": "Drop one test run by hand (design 47 G §728): the act and why are written in the hand-act log first, then the run's ask is dropped, each reader ended could not run naming the drop, and the key deleted with the run's output object.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"key": {
|
||||
"type": "string",
|
||||
"description": "the run's key in test-runs"
|
||||
},
|
||||
"why": {
|
||||
"type": "string",
|
||||
"description": "why, written with the caller the runtime proves in the hand-act log"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"key",
|
||||
"why"
|
||||
]
|
||||
},
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"run": {
|
||||
"$ref": "#/defs/TestRun",
|
||||
"description": "the entry deleted"
|
||||
},
|
||||
"ended": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/TestRunReader"
|
||||
},
|
||||
"description": "the readers it ended"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"run"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"events": [
|
||||
@@ -887,6 +974,13 @@
|
||||
"ended": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"shared": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/Shared"
|
||||
},
|
||||
"description": "the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t)"
|
||||
}
|
||||
},
|
||||
"description": "one step of a pipeline: its status in the policy and its outcome (design 47 *Pipeline steps and the policy*)",
|
||||
@@ -1494,6 +1588,17 @@
|
||||
"how": {
|
||||
"type": "string",
|
||||
"description": "for its end, how it ended: answered, ran, superseded"
|
||||
},
|
||||
"ask": {
|
||||
"type": "string",
|
||||
"description": "the step's own ask, where it has one (design 47 G §731)"
|
||||
},
|
||||
"shared": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/Shared"
|
||||
},
|
||||
"description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)"
|
||||
}
|
||||
},
|
||||
"description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283)",
|
||||
@@ -1577,6 +1682,159 @@
|
||||
"what",
|
||||
"at"
|
||||
]
|
||||
},
|
||||
"Shared": {
|
||||
"type": "object",
|
||||
"description": "a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself",
|
||||
"properties": {
|
||||
"machine": {
|
||||
"type": "string",
|
||||
"description": "the machine composed, for a composition run"
|
||||
},
|
||||
"run": {
|
||||
"type": "string",
|
||||
"description": "the run's key: in the assigner's composition-runs, or the runner's test-runs"
|
||||
},
|
||||
"ask": {
|
||||
"type": "string",
|
||||
"description": "the id of the run's ask of the build seat"
|
||||
},
|
||||
"asked-by": {
|
||||
"type": "object",
|
||||
"description": "the step that asked the run, for a test step reading another's run",
|
||||
"properties": {
|
||||
"pipeline": {
|
||||
"type": "string",
|
||||
"description": "its pipeline's key"
|
||||
},
|
||||
"artifact": {
|
||||
"type": "string",
|
||||
"description": "its artifact, <module>:<artifact>"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"pipeline",
|
||||
"artifact"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"run",
|
||||
"ask"
|
||||
]
|
||||
},
|
||||
"TestRunReader": {
|
||||
"type": "object",
|
||||
"description": "a test step reading a test run (design 47 G §725)",
|
||||
"properties": {
|
||||
"pipeline": {
|
||||
"type": "string",
|
||||
"description": "its pipeline's key"
|
||||
},
|
||||
"artifact": {
|
||||
"type": "string",
|
||||
"description": "its artifact, <module>:<artifact>"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"pipeline",
|
||||
"artifact"
|
||||
]
|
||||
},
|
||||
"TestRunPackage": {
|
||||
"type": "object",
|
||||
"description": "one package a test run ran, with its digest (design 47 G §724)",
|
||||
"properties": {
|
||||
"package": {
|
||||
"type": "string"
|
||||
},
|
||||
"digest": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"package",
|
||||
"digest"
|
||||
]
|
||||
},
|
||||
"TestRun": {
|
||||
"type": "object",
|
||||
"description": "one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u)",
|
||||
"properties": {
|
||||
"key": {
|
||||
"type": "string",
|
||||
"description": "the key, <candidate key>.<level>.<digest>, each part escaped as a key's name"
|
||||
},
|
||||
"level": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"lint",
|
||||
"unit",
|
||||
"integration"
|
||||
]
|
||||
},
|
||||
"digest": {
|
||||
"type": "string",
|
||||
"description": "the share digest (design 47 G §724)"
|
||||
},
|
||||
"seat": {
|
||||
"type": "string",
|
||||
"description": "the build seat asked"
|
||||
},
|
||||
"ask": {
|
||||
"type": "string",
|
||||
"description": "the id of the build seat's ask"
|
||||
},
|
||||
"node": {
|
||||
"type": "string",
|
||||
"description": "the node whose holder took the ask"
|
||||
},
|
||||
"started": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"readers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/TestRunReader"
|
||||
},
|
||||
"description": "the steps reading the run"
|
||||
},
|
||||
"ended": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"outcome": {
|
||||
"type": "string"
|
||||
},
|
||||
"said": {
|
||||
"type": "string",
|
||||
"description": "the outcome's words"
|
||||
},
|
||||
"tests": {
|
||||
"$ref": "#/defs/TestCounts"
|
||||
},
|
||||
"output": {
|
||||
"type": "string",
|
||||
"description": "the run's output object, by its name in test-runs-objects"
|
||||
},
|
||||
"packages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/TestRunPackage"
|
||||
},
|
||||
"description": "the packages it ran, read from its output object"
|
||||
},
|
||||
"gone": {
|
||||
"type": "string",
|
||||
"description": "why the packages are not named: the run's output is gone"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"key",
|
||||
"level",
|
||||
"digest"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,5 +284,54 @@
|
||||
"since": "2026-11-11T10:00:00Z",
|
||||
"how": "go-on"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1/AC1 a reading composition step ended, naming per machine the run it read and its ask",
|
||||
"of": "event:step-ended",
|
||||
"subject": "mesh.seat.mesh-wants.event.step-ended.laptop",
|
||||
"given": {
|
||||
"v": 1,
|
||||
"id": "m2kx9a1b3c4d6c",
|
||||
"pipeline": "main.12.nats",
|
||||
"step": "composition",
|
||||
"module": "nats",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"fingerprint": "fp-0123",
|
||||
"node": "laptop",
|
||||
"outcome": "passed",
|
||||
"warnings": [
|
||||
"anchor holds xclip: xclip and demo both own /etc/x"
|
||||
],
|
||||
"shared": [
|
||||
{
|
||||
"machine": "anchor",
|
||||
"run": "main.12.anchor",
|
||||
"ask": "r1a2b3"
|
||||
},
|
||||
{
|
||||
"machine": "home-server",
|
||||
"run": "main.12.home-server",
|
||||
"ask": "r4d5e6"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1/AC1 a composition step's progress, naming the runs it waits on",
|
||||
"of": "event:step-progress",
|
||||
"subject": "mesh.seat.mesh-wants.event.step-progress.anchor",
|
||||
"given": {
|
||||
"asker": "assigner",
|
||||
"ask": "a1b2",
|
||||
"phase": "run",
|
||||
"at": "2026-11-17T10:01:00Z",
|
||||
"shared": [
|
||||
{
|
||||
"machine": "anchor",
|
||||
"run": "main.12.anchor",
|
||||
"ask": "r1a2b3"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -175,5 +175,109 @@
|
||||
"chunks": 3,
|
||||
"digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1 compositions answers each run with its readers, verdict and the fill",
|
||||
"of": "verb:compositions.answer",
|
||||
"given": {
|
||||
"runs": [
|
||||
{
|
||||
"key": "main.12.anchor",
|
||||
"candidate": "main.12",
|
||||
"machine": "anchor",
|
||||
"program": {
|
||||
"repository": "novox/mesh-host",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"directory": "cmd/mesh-compose",
|
||||
"fingerprint": "src2:aaaa"
|
||||
},
|
||||
"input": "sha256:0123",
|
||||
"ask": "r1a2b3",
|
||||
"started": "2026-11-17T10:00:00Z",
|
||||
"readers": [
|
||||
{
|
||||
"id": "m2kx9a1b3c4d6c",
|
||||
"pipeline": "main.12.nats",
|
||||
"step": "composition",
|
||||
"module": "nats",
|
||||
"branch": "main",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"revision": 12,
|
||||
"fingerprint": "fp-0123"
|
||||
}
|
||||
],
|
||||
"ended": "2026-11-17T10:03:00Z",
|
||||
"outcome": "passed",
|
||||
"verdict": {
|
||||
"composes": true,
|
||||
"blamed": []
|
||||
}
|
||||
}
|
||||
],
|
||||
"fill": {
|
||||
"store": "composition-runs",
|
||||
"bytes": 4096,
|
||||
"cap": 16777216
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1 compositions-drop takes a candidate, a machine and why",
|
||||
"of": "verb:compositions-drop.input",
|
||||
"given": {
|
||||
"candidate": "main.12",
|
||||
"machine": "anchor",
|
||||
"why": "its ask never came back"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "C1 compositions-drop answers the run it deleted and the readers it ended",
|
||||
"of": "verb:compositions-drop.answer",
|
||||
"given": {
|
||||
"run": {
|
||||
"key": "main.12.anchor",
|
||||
"candidate": "main.12",
|
||||
"machine": "anchor",
|
||||
"program": {
|
||||
"repository": "novox/mesh-host",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"directory": "cmd/mesh-compose",
|
||||
"fingerprint": "src2:aaaa"
|
||||
},
|
||||
"input": "sha256:0123",
|
||||
"ask": "r1a2b3",
|
||||
"started": "2026-11-17T10:00:00Z",
|
||||
"readers": [
|
||||
{
|
||||
"id": "m2kx9a1b3c4d6c",
|
||||
"pipeline": "main.12.nats",
|
||||
"step": "composition",
|
||||
"module": "nats",
|
||||
"branch": "main",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"revision": 12,
|
||||
"fingerprint": "fp-0123"
|
||||
}
|
||||
],
|
||||
"ended": "2026-11-17T10:03:00Z",
|
||||
"outcome": "passed",
|
||||
"verdict": {
|
||||
"composes": true,
|
||||
"blamed": []
|
||||
}
|
||||
},
|
||||
"ended": [
|
||||
{
|
||||
"id": "m2kx9a1b3c4d6c",
|
||||
"pipeline": "main.12.nats",
|
||||
"step": "composition",
|
||||
"module": "nats",
|
||||
"branch": "main",
|
||||
"commit": "0123456789abcdef0123456789abcdef01234567",
|
||||
"revision": 12,
|
||||
"fingerprint": "fp-0123"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -202,6 +202,86 @@
|
||||
]
|
||||
},
|
||||
"chunks": true
|
||||
},
|
||||
{
|
||||
"name": "compositions",
|
||||
"class": "read",
|
||||
"description": "Each composition run the assigner keeps (design 47 G \u00a7728): its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"candidate": {
|
||||
"type": "string",
|
||||
"description": "the candidate's key"
|
||||
},
|
||||
"machine": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"runs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/CompositionRun"
|
||||
}
|
||||
},
|
||||
"fill": {
|
||||
"$ref": "#/defs/Fill"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"runs",
|
||||
"fill"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "compositions-drop",
|
||||
"class": "act",
|
||||
"description": "Drop one composition run by hand (design 47 G \u00a7728): the act and why are written in the hand-act log first, then the run's ask is dropped, each reader ended could not run naming the drop, and the key deleted.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"candidate": {
|
||||
"type": "string",
|
||||
"description": "the candidate's key"
|
||||
},
|
||||
"machine": {
|
||||
"type": "string"
|
||||
},
|
||||
"why": {
|
||||
"type": "string",
|
||||
"description": "why, written with the caller the runtime proves in the hand-act log"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"candidate",
|
||||
"machine",
|
||||
"why"
|
||||
]
|
||||
},
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"run": {
|
||||
"$ref": "#/defs/CompositionRun",
|
||||
"description": "the entry deleted"
|
||||
},
|
||||
"ended": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/CompositionReader"
|
||||
},
|
||||
"description": "the readers it ended"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"run"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"accepts": [
|
||||
@@ -470,6 +550,13 @@
|
||||
"at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"shared": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/Shared"
|
||||
},
|
||||
"description": "the composition runs the step waits on, by machine, each with its ask (design 47 G \u00a7723, \u00a7729)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -808,22 +895,6 @@
|
||||
"description": "an entry, written once, removed when no kept revision lists it"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "inputs",
|
||||
"template": "state",
|
||||
"history": 1,
|
||||
"description": "a step's input, named by its write time and its sha256; a value over one chunk is held in the state's objects store (design 47 G sections 354, 712)",
|
||||
"key-forms": [
|
||||
{
|
||||
"key": "<time>.<digest>",
|
||||
"value": {
|
||||
"type": "string",
|
||||
"description": "the input's bytes, base64"
|
||||
},
|
||||
"description": "one input"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"defs": {
|
||||
@@ -1434,6 +1505,13 @@
|
||||
"ended": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"shared": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/Shared"
|
||||
},
|
||||
"description": "per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G \u00a7723, \u00a7729)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -1488,6 +1566,195 @@
|
||||
"required": [
|
||||
"ask"
|
||||
]
|
||||
},
|
||||
"Shared": {
|
||||
"type": "object",
|
||||
"description": "a run whose result a step took or waits on, asked once for every step that reads it (design 47 G \u00a7729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself",
|
||||
"properties": {
|
||||
"machine": {
|
||||
"type": "string",
|
||||
"description": "the machine composed, for a composition run"
|
||||
},
|
||||
"run": {
|
||||
"type": "string",
|
||||
"description": "the run's key: in the assigner's composition-runs, or the runner's test-runs"
|
||||
},
|
||||
"ask": {
|
||||
"type": "string",
|
||||
"description": "the id of the run's ask of the build seat"
|
||||
},
|
||||
"asked-by": {
|
||||
"type": "object",
|
||||
"description": "the step that asked the run, for a test step reading another's run",
|
||||
"properties": {
|
||||
"pipeline": {
|
||||
"type": "string",
|
||||
"description": "its pipeline's key"
|
||||
},
|
||||
"artifact": {
|
||||
"type": "string",
|
||||
"description": "its artifact, <module>:<artifact>"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"pipeline",
|
||||
"artifact"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"run",
|
||||
"ask"
|
||||
]
|
||||
},
|
||||
"CompositionReader": {
|
||||
"type": "object",
|
||||
"description": "a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G \u00a7719, A-468)",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string",
|
||||
"description": "the step's ask's id"
|
||||
},
|
||||
"pipeline": {
|
||||
"type": "string",
|
||||
"description": "the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is"
|
||||
},
|
||||
"step": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"composition"
|
||||
]
|
||||
},
|
||||
"module": {
|
||||
"type": "string"
|
||||
},
|
||||
"branch": {
|
||||
"type": "string",
|
||||
"description": "the watched branch, or the branch of a run on request"
|
||||
},
|
||||
"commit": {
|
||||
"type": "string",
|
||||
"description": "the commit the step runs at"
|
||||
},
|
||||
"revision": {
|
||||
"type": "integer",
|
||||
"description": "the catalog graph revision of the branch the pipeline runs for"
|
||||
},
|
||||
"source": {
|
||||
"type": "integer",
|
||||
"description": "the source revision of the watched branch; absent for a run on request"
|
||||
},
|
||||
"fingerprint": {
|
||||
"type": "string",
|
||||
"description": "the module's build-input fingerprint at the commit"
|
||||
},
|
||||
"run": {
|
||||
"type": "string",
|
||||
"description": "the run on request's id: its composition writes no want (ADR 0666 section 18)"
|
||||
},
|
||||
"rerun": {
|
||||
"type": "boolean",
|
||||
"description": "the step is asked again by the runner's rerun"
|
||||
},
|
||||
"by": {
|
||||
"type": "string",
|
||||
"description": "for a rerun, the caller the runtime proved"
|
||||
},
|
||||
"ended": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"pipeline",
|
||||
"step",
|
||||
"module",
|
||||
"branch",
|
||||
"commit",
|
||||
"fingerprint"
|
||||
]
|
||||
},
|
||||
"CompositionRun": {
|
||||
"type": "object",
|
||||
"description": "one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G \u00a7719, \u00a7728; A-469u)",
|
||||
"properties": {
|
||||
"key": {
|
||||
"type": "string",
|
||||
"description": "the key, <candidate key>.<machine>, each part escaped as a key's name"
|
||||
},
|
||||
"candidate": {
|
||||
"type": "string",
|
||||
"description": "the candidate's key: <branch>.<revision>, or run-<id> for a run on request"
|
||||
},
|
||||
"machine": {
|
||||
"type": "string"
|
||||
},
|
||||
"program": {
|
||||
"type": "object",
|
||||
"description": "the composer program, chosen once when the run started",
|
||||
"properties": {
|
||||
"repository": {
|
||||
"type": "string"
|
||||
},
|
||||
"commit": {
|
||||
"type": "string"
|
||||
},
|
||||
"directory": {
|
||||
"type": "string"
|
||||
},
|
||||
"fingerprint": {
|
||||
"type": "string",
|
||||
"description": "the node-engine fingerprint it was chosen from"
|
||||
}
|
||||
}
|
||||
},
|
||||
"input": {
|
||||
"type": "string",
|
||||
"description": "the digest of the kept request"
|
||||
},
|
||||
"ask": {
|
||||
"type": "string",
|
||||
"description": "the id of the build seat's run ask"
|
||||
},
|
||||
"started": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"readers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/CompositionReader"
|
||||
},
|
||||
"description": "the composition steps reading the run"
|
||||
},
|
||||
"ended": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"outcome": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"passed",
|
||||
"failed",
|
||||
"could not run"
|
||||
]
|
||||
},
|
||||
"said": {
|
||||
"type": "string",
|
||||
"description": "the outcome's words"
|
||||
},
|
||||
"verdict": {
|
||||
"type": "object",
|
||||
"description": "that machine's composer verdict (design 47 G \u00a7721)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"key",
|
||||
"candidate",
|
||||
"machine"
|
||||
]
|
||||
}
|
||||
},
|
||||
"acts": [
|
||||
|
||||
@@ -17,6 +17,8 @@ export declare const verbs: {
|
||||
readonly accept: "accept";
|
||||
readonly cancel: "cancel";
|
||||
readonly askAgain: "ask-again";
|
||||
readonly testRuns: "test-runs";
|
||||
readonly testRunsDrop: "test-runs-drop";
|
||||
};
|
||||
|
||||
export declare const events: {
|
||||
@@ -316,6 +318,26 @@ export interface RunStarted {
|
||||
"v": number;
|
||||
}
|
||||
|
||||
/** the step that asked the run, for a test step reading another's run */
|
||||
export interface SharedAskedBy {
|
||||
/** its artifact, <module>:<artifact> */
|
||||
"artifact": string;
|
||||
/** its pipeline's key */
|
||||
"pipeline": string;
|
||||
}
|
||||
|
||||
/** a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself */
|
||||
export interface Shared {
|
||||
/** the id of the run's ask of the build seat */
|
||||
"ask": string;
|
||||
/** the step that asked the run, for a test step reading another's run */
|
||||
"asked-by"?: SharedAskedBy;
|
||||
/** the machine composed, for a composition run */
|
||||
"machine"?: string;
|
||||
/** the run's key: in the assigner's composition-runs, or the runner's test-runs */
|
||||
"run": string;
|
||||
}
|
||||
|
||||
/** a step's start or end: the step, its status, its outcome, its fingerprint, its test counts, what a carried result was carried from, and for a rerun that it is one and its proved caller; a bypassed step too */
|
||||
export interface StepEvent {
|
||||
"at"?: string;
|
||||
@@ -366,6 +388,8 @@ export interface StepResult {
|
||||
"rerun"?: boolean;
|
||||
/** the seat that serves the step, whose holder alone may end it (ADR 0666 section 21) */
|
||||
"seat"?: string;
|
||||
/** the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t) */
|
||||
"shared"?: Shared[];
|
||||
"started"?: string;
|
||||
/** the step's status in the module's policy */
|
||||
"status": "required" | "optional" | "bypass";
|
||||
@@ -381,6 +405,49 @@ export interface TestCounts {
|
||||
"unit"?: number;
|
||||
}
|
||||
|
||||
/** one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u) */
|
||||
export interface TestRun {
|
||||
/** the id of the build seat's ask */
|
||||
"ask"?: string;
|
||||
/** the share digest (design 47 G §724) */
|
||||
"digest": string;
|
||||
"ended"?: string;
|
||||
/** why the packages are not named: the run's output is gone */
|
||||
"gone"?: string;
|
||||
/** the key, <candidate key>.<level>.<digest>, each part escaped as a key's name */
|
||||
"key": string;
|
||||
"level": "lint" | "unit" | "integration";
|
||||
/** the node whose holder took the ask */
|
||||
"node"?: string;
|
||||
"outcome"?: string;
|
||||
/** the run's output object, by its name in test-runs-objects */
|
||||
"output"?: string;
|
||||
/** the packages it ran, read from its output object */
|
||||
"packages"?: TestRunPackage[];
|
||||
/** the steps reading the run */
|
||||
"readers"?: TestRunReader[];
|
||||
/** the outcome's words */
|
||||
"said"?: string;
|
||||
/** the build seat asked */
|
||||
"seat"?: string;
|
||||
"started"?: string;
|
||||
"tests"?: TestCounts;
|
||||
}
|
||||
|
||||
/** one package a test run ran, with its digest (design 47 G §724) */
|
||||
export interface TestRunPackage {
|
||||
"digest": string;
|
||||
"package": string;
|
||||
}
|
||||
|
||||
/** a test step reading a test run (design 47 G §725) */
|
||||
export interface TestRunReader {
|
||||
/** its artifact, <module>:<artifact> */
|
||||
"artifact": string;
|
||||
/** its pipeline's key */
|
||||
"pipeline": string;
|
||||
}
|
||||
|
||||
/** the toolchain a build was made with: the language, the installed version and the build module's recipe */
|
||||
export interface ToolchainUsed {
|
||||
"language": string;
|
||||
@@ -391,6 +458,8 @@ export interface ToolchainUsed {
|
||||
/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283) */
|
||||
export interface Wait {
|
||||
"artifact"?: string;
|
||||
/** the step's own ask, where it has one (design 47 G §731) */
|
||||
"ask"?: string;
|
||||
/** the watched branch, or the branch of a run on request */
|
||||
"branch": string;
|
||||
/** the commit the module's entry was last changed at */
|
||||
@@ -410,6 +479,8 @@ export interface Wait {
|
||||
"revision"?: number;
|
||||
/** the run on request's id, for a pipeline of a run; such a pipeline delivers nothing */
|
||||
"run"?: string;
|
||||
/** the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v) */
|
||||
"shared"?: Shared[];
|
||||
"since": string;
|
||||
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
|
||||
"source"?: number;
|
||||
@@ -594,6 +665,37 @@ export interface AskAgainAnswer {
|
||||
"result": StepResult;
|
||||
}
|
||||
|
||||
/** is what test-runs takes. */
|
||||
export interface TestRunsInput {
|
||||
/** the candidate's key */
|
||||
"candidate"?: string;
|
||||
"digest"?: string;
|
||||
"level"?: "lint" | "unit" | "integration";
|
||||
"module"?: string;
|
||||
}
|
||||
|
||||
/** is what test-runs answers. */
|
||||
export interface TestRunsAnswer {
|
||||
"fill": Fill;
|
||||
"runs": TestRun[];
|
||||
}
|
||||
|
||||
/** is what test-runs-drop takes. */
|
||||
export interface TestRunsDropInput {
|
||||
/** the run's key in test-runs */
|
||||
"key": string;
|
||||
/** why, written with the caller the runtime proves in the hand-act log */
|
||||
"why": string;
|
||||
}
|
||||
|
||||
/** is what test-runs-drop answers. */
|
||||
export interface TestRunsDropAnswer {
|
||||
/** the readers it ended */
|
||||
"ended"?: TestRunReader[];
|
||||
/** the entry deleted */
|
||||
"run": TestRun;
|
||||
}
|
||||
|
||||
/** is the body of the event pipeline-started. */
|
||||
export type PipelineStartedEvent = PipelineStarted;
|
||||
|
||||
|
||||
@@ -17,6 +17,8 @@ export const verbs = {
|
||||
accept: "accept",
|
||||
cancel: "cancel",
|
||||
askAgain: "ask-again",
|
||||
testRuns: "test-runs",
|
||||
testRunsDrop: "test-runs-drop",
|
||||
};
|
||||
|
||||
export const events = {
|
||||
|
||||
+117
-10
@@ -11,6 +11,8 @@ export declare const verbs: {
|
||||
readonly entry: "entry";
|
||||
readonly difference: "difference";
|
||||
readonly input: "input";
|
||||
readonly compositions: "compositions";
|
||||
readonly compositionsDrop: "compositions-drop";
|
||||
};
|
||||
|
||||
export declare const accepts: {
|
||||
@@ -78,7 +80,6 @@ export declare const states: {
|
||||
readonly differences: "differences";
|
||||
readonly compositions: "compositions";
|
||||
readonly assignmentRevisions: "assignment-revisions";
|
||||
readonly inputs: "inputs";
|
||||
};
|
||||
|
||||
/** The key of one value of the state wants under the form "want.<node>.<module>". */
|
||||
@@ -171,12 +172,6 @@ export declare function assignmentRevisionsByEntryDigestKey(digest: string): str
|
||||
/** The parts of a key of the form "entry.<digest>", or undefined for a key of another form. */
|
||||
export declare function parseAssignmentRevisionsByEntryDigestKey(_key: string): { digest: string } | undefined;
|
||||
|
||||
/** The key of one value of the state inputs under the form "<time>.<digest>". */
|
||||
export declare function inputsByTimeDigestKey(time: string, digest: string): string;
|
||||
|
||||
/** The parts of a key of the form "<time>.<digest>", or undefined for a key of another form. */
|
||||
export declare function parseInputsByTimeDigestKey(_key: string): { time: string; digest: string } | undefined;
|
||||
|
||||
/** an ask the assigner puts to a person: the router's ask whole, as the assigner composes it, and how many times it was written at its key */
|
||||
export interface Ask {
|
||||
/** the seat operator-channel's ask, whole */
|
||||
@@ -265,6 +260,67 @@ export interface Change {
|
||||
"to"?: string;
|
||||
}
|
||||
|
||||
/** a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G §719, A-468) */
|
||||
export interface CompositionReader {
|
||||
/** the watched branch, or the branch of a run on request */
|
||||
"branch": string;
|
||||
/** for a rerun, the caller the runtime proved */
|
||||
"by"?: string;
|
||||
/** the commit the step runs at */
|
||||
"commit": string;
|
||||
/** when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m) */
|
||||
"ended"?: string;
|
||||
/** the module's build-input fingerprint at the commit */
|
||||
"fingerprint": string;
|
||||
/** the step's ask's id */
|
||||
"id": string;
|
||||
"module": string;
|
||||
/** the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is */
|
||||
"pipeline": string;
|
||||
/** the step is asked again by the runner's rerun */
|
||||
"rerun"?: boolean;
|
||||
/** the catalog graph revision of the branch the pipeline runs for */
|
||||
"revision"?: number;
|
||||
/** the run on request's id: its composition writes no want (ADR 0666 section 18) */
|
||||
"run"?: string;
|
||||
/** the source revision of the watched branch; absent for a run on request */
|
||||
"source"?: number;
|
||||
"step": "composition";
|
||||
}
|
||||
|
||||
/** the composer program, chosen once when the run started */
|
||||
export interface CompositionRunProgram {
|
||||
"commit"?: string;
|
||||
"directory"?: string;
|
||||
/** the node-engine fingerprint it was chosen from */
|
||||
"fingerprint"?: string;
|
||||
"repository"?: string;
|
||||
}
|
||||
|
||||
/** one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G §719, §728; A-469u) */
|
||||
export interface CompositionRun {
|
||||
/** the id of the build seat's run ask */
|
||||
"ask"?: string;
|
||||
/** the candidate's key: <branch>.<revision>, or run-<id> for a run on request */
|
||||
"candidate": string;
|
||||
"ended"?: string;
|
||||
/** the digest of the kept request */
|
||||
"input"?: string;
|
||||
/** the key, <candidate key>.<machine>, each part escaped as a key's name */
|
||||
"key": string;
|
||||
"machine": string;
|
||||
"outcome"?: "passed" | "failed" | "could not run";
|
||||
/** the composer program, chosen once when the run started */
|
||||
"program"?: CompositionRunProgram;
|
||||
/** the composition steps reading the run */
|
||||
"readers"?: CompositionReader[];
|
||||
/** the outcome's words */
|
||||
"said"?: string;
|
||||
"started"?: string;
|
||||
/** that machine's composer verdict (design 47 G §721) */
|
||||
"verdict"?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface DifferenceRefusedItem {
|
||||
"module": string;
|
||||
"why": string;
|
||||
@@ -340,6 +396,26 @@ export interface NodeEntryRef {
|
||||
"node": string;
|
||||
}
|
||||
|
||||
/** the step that asked the run, for a test step reading another's run */
|
||||
export interface SharedAskedBy {
|
||||
/** its artifact, <module>:<artifact> */
|
||||
"artifact": string;
|
||||
/** its pipeline's key */
|
||||
"pipeline": string;
|
||||
}
|
||||
|
||||
/** a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself */
|
||||
export interface Shared {
|
||||
/** the id of the run's ask of the build seat */
|
||||
"ask": string;
|
||||
/** the step that asked the run, for a test step reading another's run */
|
||||
"asked-by"?: SharedAskedBy;
|
||||
/** the machine composed, for a composition run */
|
||||
"machine"?: string;
|
||||
/** the run's key: in the assigner's composition-runs, or the runner's test-runs */
|
||||
"run": string;
|
||||
}
|
||||
|
||||
/** the composition step's end, said by the assigner on the node it runs on (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at */
|
||||
export interface StepEnded {
|
||||
/** for a rerun, the caller the runtime proved */
|
||||
@@ -358,6 +434,8 @@ export interface StepEnded {
|
||||
"rerun"?: boolean;
|
||||
/** why it failed or could not run: the two modules owning one path, the part not heard */
|
||||
"said"?: string;
|
||||
/** per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G §723, §729) */
|
||||
"shared"?: Shared[];
|
||||
"started"?: string;
|
||||
"step": "composition";
|
||||
/** the layout version */
|
||||
@@ -506,6 +584,36 @@ export interface InputAnswer {
|
||||
"digest": string;
|
||||
}
|
||||
|
||||
/** is what compositions takes. */
|
||||
export interface CompositionsInput {
|
||||
/** the candidate's key */
|
||||
"candidate"?: string;
|
||||
"machine"?: string;
|
||||
}
|
||||
|
||||
/** is what compositions answers. */
|
||||
export interface CompositionsAnswer {
|
||||
"fill": Fill;
|
||||
"runs": CompositionRun[];
|
||||
}
|
||||
|
||||
/** is what compositions-drop takes. */
|
||||
export interface CompositionsDropInput {
|
||||
/** the candidate's key */
|
||||
"candidate": string;
|
||||
"machine": string;
|
||||
/** why, written with the caller the runtime proves in the hand-act log */
|
||||
"why": string;
|
||||
}
|
||||
|
||||
/** is what compositions-drop answers. */
|
||||
export interface CompositionsDropAnswer {
|
||||
/** the readers it ended */
|
||||
"ended"?: CompositionReader[];
|
||||
/** the entry deleted */
|
||||
"run": CompositionRun;
|
||||
}
|
||||
|
||||
/** is the body of step. */
|
||||
export type StepAccept = StepRun;
|
||||
|
||||
@@ -589,6 +697,8 @@ export interface StepProgressEvent {
|
||||
"asker": string;
|
||||
"at": string;
|
||||
"phase": "fetch" | "build" | "run" | "report";
|
||||
/** the composition runs the step waits on, by machine, each with its ask (design 47 G §723, §729) */
|
||||
"shared"?: Shared[];
|
||||
}
|
||||
|
||||
/** is one value of the state wants under the key want.<node>.<module>. */
|
||||
@@ -692,6 +802,3 @@ export type AssignmentRevisionsByCandidateGroupHeadsValue = AssignmentRevision;
|
||||
/** is one value of the state assignment-revisions under the key entry.<digest>. */
|
||||
export type AssignmentRevisionsByEntryDigestValue = AssignmentEntry;
|
||||
|
||||
/** is one value of the state inputs under the key <time>.<digest>. */
|
||||
export type InputsByTimeDigestValue = string;
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ export const verbs = {
|
||||
entry: "entry",
|
||||
difference: "difference",
|
||||
input: "input",
|
||||
compositions: "compositions",
|
||||
compositionsDrop: "compositions-drop",
|
||||
};
|
||||
|
||||
export const accepts = {
|
||||
@@ -84,7 +86,6 @@ export const states = {
|
||||
differences: "differences",
|
||||
compositions: "compositions",
|
||||
assignmentRevisions: "assignment-revisions",
|
||||
inputs: "inputs",
|
||||
};
|
||||
|
||||
/** The key of one value of the state wants under the form "want.<node>.<module>". */
|
||||
@@ -403,22 +404,3 @@ export function parseAssignmentRevisionsByEntryDigestKey(_key) {
|
||||
return { digest };
|
||||
}
|
||||
|
||||
/** The key of one value of the state inputs under the form "<time>.<digest>". */
|
||||
export function inputsByTimeDigestKey(time, digest) {
|
||||
return time + "." + digest;
|
||||
}
|
||||
|
||||
/** The parts of a key of the form "<time>.<digest>", or undefined for a key of another form. */
|
||||
export function parseInputsByTimeDigestKey(_key) {
|
||||
let _rest = _key;
|
||||
const _at0 = _rest.indexOf(".");
|
||||
if (_at0 <= 0) return undefined;
|
||||
const time = _rest.slice(0, _at0);
|
||||
_rest = _rest.slice(_at0 + 1);
|
||||
if (_rest === "") return undefined;
|
||||
const digest = _rest;
|
||||
_rest = "";
|
||||
if (_rest !== "") return undefined;
|
||||
return { time, digest };
|
||||
}
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ func as[T any](args map[string]any) (T, error) {
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// Tools are the runner's read verbs over its states pipelines and kept.
|
||||
// Tools are the runner's read verbs over its states pipelines and kept; test-runs is the process's (A-470i).
|
||||
func Tools(r pipeline.Reads) []stdio.Tool {
|
||||
runs := map[string]func(map[string]any) (any, error){
|
||||
seat.VerbShow: func(a map[string]any) (any, error) {
|
||||
@@ -152,6 +152,7 @@ func main() {
|
||||
tools = append(tools, UnloggedTools(stdio.State(seat.StatePipelines), stdio.Log(seat.StateStepLog), meshhandactsclient.Recorder{Ask: stdio.Ask},
|
||||
func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] "+f+"\n", a...) })...)
|
||||
tools = append(tools, CancelTool(r.List, stdio.Ask), RerunTool(stdio.Ask))
|
||||
tools = append(tools, TestRunsTools(stdio.Ask)...)
|
||||
tools = append(tools, HealthTool(r.Pipelines, kept{stdio.State(pipeline.StateConsumers)}, time.Now))
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-pipeline-tools] %v\n", err)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// TestRunsTools are runner_test_runs and runner_test_runs_drop (design 47 G §728; A-469f, A-469u): each calls its seat verb
|
||||
// over the bus, both answered by the runner's process, which holds test-runs-cap-mib (A-469g, A-470i).
|
||||
func TestRunsTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: Prefix + "_test_runs",
|
||||
Description: "The pipeline runner's test runs, one per candidate, level and tested packages (the state test-runs): each with its packages read from its output object, its ask, node, readers, outcome and counts, and the state's fill against its cap setting test-runs-cap-mib; filtered by a candidate, a level, a digest or a module, or none.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"candidate": map[string]any{"type": "string"}, "level": map[string]any{"type": "string"},
|
||||
"digest": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
in, err := as[seat.TestRunsInput](args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ask("seat:"+seat.Seat+"."+seat.VerbTestRuns, in)
|
||||
}},
|
||||
{Name: Prefix + "_test_runs_drop",
|
||||
Description: "Drop one test run by the seat verb test-runs-drop, a hand act recorded first: its ask is dropped, each reading step ended could not run naming the drop, and its key and output object deleted. It takes the run's key and why.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"key": map[string]any{"type": "string"}, "why": map[string]any{"type": "string"}}, "required": []string{"key", "why"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
in, err := as[seat.TestRunsDropInput](args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(in.Key) == "" || strings.TrimSpace(in.Why) == "" {
|
||||
return nil, errors.New("test-runs-drop names the run's key and why")
|
||||
}
|
||||
return ask("seat:"+seat.Seat+"."+seat.VerbTestRunsDrop, in)
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,7 @@ func deleteBound(ctx context.Context, bounds jetstream.KeyValue) func(key string
|
||||
// armDropBound arms the five-minute bound of a drop, a key of the state bounds with its own time to live (A-77, A-222).
|
||||
func armDropBound(ctx context.Context, bounds jetstream.KeyValue) func(key string) error {
|
||||
return func(key string) error {
|
||||
_, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.AskBound))
|
||||
_, err := bounds.Create(ctx, key, []byte(`{"drop":"`+key+`"}`), jetstream.KeyTTL(pipeline.DropBound))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return nil
|
||||
}
|
||||
@@ -67,26 +67,19 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func
|
||||
return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning,
|
||||
Summary: fmt.Sprintf("the call %s of ask %s is unanswered", call, ask),
|
||||
Headline: "A call whose answer is kept is not answered",
|
||||
Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.AskBound),
|
||||
Explanation: fmt.Sprintf("The drop of the superseded step's ask %s was asked of the seat %s as a kept call and no holder answered it within %s, so the build it names may still run.", ask, buildSeat, pipeline.DropBound),
|
||||
Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: id}})
|
||||
}
|
||||
}
|
||||
|
||||
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered, an ask's ends the step no
|
||||
// holder took (§32, §535, A-77); another key is none of the runner's.
|
||||
func boundExpired(h *pipeline.Hearer, unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
|
||||
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none
|
||||
// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them).
|
||||
func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
|
||||
return func(key string) {
|
||||
if s, ask, ok := pipeline.ParseDropBound(key); ok {
|
||||
if err := unanswered(s, ask); err != nil {
|
||||
logf("the drop of ask %s on %s, unanswered, is not said: %v", ask, s, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !pipeline.IsAskKey(key) {
|
||||
return
|
||||
}
|
||||
if err := h.NotTaken(key, pipeline.AskBound); err != nil {
|
||||
logf("the step of ask %s, which no holder took, is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ func (f *fakeFindings) Clear(_ context.Context, scope, id, kind, _ string) error
|
||||
func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) {
|
||||
w := &fakeFindings{}
|
||||
var logged int
|
||||
expired := boundExpired(nil, dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ })
|
||||
expired := boundExpired(dropUnanswered(context.Background(), w, "the-runner", func() time.Time { return time.Unix(100, 0) }), func(string, ...any) { logged++ })
|
||||
expired("not-a-bound")
|
||||
if len(w.written) != 0 {
|
||||
t.Fatalf("a key of no drop was said: %+v", w.written)
|
||||
|
||||
@@ -67,8 +67,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The bound of each ask a build seat's holder must take: a key with its own time to live, its expiry heard (§535, A-77).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound an ask no holder takes")
|
||||
// The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -151,14 +151,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// for its step.
|
||||
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
|
||||
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
|
||||
Arm: armBound(ctx, bounds), Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has}
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
placeHalf := func(ev catalog.CatalogRevisionEvent, half pipeline.Half) error {
|
||||
n, err := starter.PlaceRun(ev, half, rv.Catalog.View)
|
||||
logger.Printf("the candidate %s's revision %d placed %d pipelines of its run", ev.Group, ev.Revision, n)
|
||||
return err
|
||||
}
|
||||
resume := func(branch string) {
|
||||
rv.Holders.refresh() // a half held may name the claimant of the seat-holders view (§647)
|
||||
half, ok := rv.Catalog.View(branch)
|
||||
if !ok {
|
||||
return
|
||||
@@ -182,6 +183,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
resumer.Store(&resume)
|
||||
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
|
||||
advance := func(branch string) {
|
||||
rv.Holders.refresh()
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
|
||||
}
|
||||
@@ -227,23 +229,41 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// A newer commit or publish supersedes the older running pipelines it replaces, their asks dropped as kept calls (§31,
|
||||
// §350; A-182, A-202).
|
||||
starter.Supersede = func(newer seat.Pipeline) {
|
||||
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)), dropBound(ctx, bounds))
|
||||
n, err := hearer.Supersede(newer, pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings)))
|
||||
if err != nil {
|
||||
logger.Printf("the pipelines %s supersedes: %d ended, then %v", newer.Pipeline, n, err)
|
||||
} else if n > 0 {
|
||||
logger.Printf("the pipeline %s superseded %d older pipelines", newer.Pipeline, n)
|
||||
}
|
||||
}
|
||||
// The test runs (§724, §725): one run per candidate, level and digest, kept in the state test-runs with its clock.
|
||||
runDrop := pipeline.BoundDrop(dropAsk(calls, cred.Module), armDropBound(ctx, bounds), dropAnswered(ctx, deleteBound(ctx, bounds), rv.Findings))
|
||||
runsKV, runObjs, err := openTestRuns(ctx, js, cred, getenv, starter, hearer, runDrop, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := watchRunBounds(ctx, runsKV, hearer.RunExpired, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearStepProgress(heard, hearer.RunProgress, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearStepEnded(heard, hearer, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearToolchain(ctx, heard, toolchains, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hearAskTaken(ctx, heard, bounds, hearer.Taken, logger.Printf); err != nil {
|
||||
taken := func(ask, node string) error {
|
||||
if err := hearer.Taken(ask, node); err != nil {
|
||||
return err
|
||||
}
|
||||
return hearer.RunTaken(ask, node)
|
||||
}
|
||||
if err := hearAskTaken(heard, taken, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
expired := boundExpired(hearer, dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -261,6 +281,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// An entry whose step ended and was not appended is appended at the start, in order (§606, §549).
|
||||
hearer.AppendUnlogged()
|
||||
// A test run whose bound passed while the runner was away ends now, as its expiry would have ended it (§725).
|
||||
if n, err := hearer.TakeUp(); err != nil || n > 0 {
|
||||
logger.Printf("the test runs taken up at the start: %d ended past their bound (%v)", n, err)
|
||||
}
|
||||
go whileTheLogIsFull(ctx, fills, hearer, time.Minute)
|
||||
// The seat list, read over the calls connection and heard on the hub (seatlist.Part), which the seat's write verbs check
|
||||
// their callers against (A-186): registered before the hub starts, read again once its events are live.
|
||||
@@ -273,7 +297,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24).
|
||||
logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred))
|
||||
// The seat's write verbs are answered here, by the process that holds the pipelines (A-107).
|
||||
stopVerbs, err := answerVerbs(calls, seats, writeVerbs(ctx, hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls, bounds))
|
||||
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
|
||||
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
|
||||
stopVerbs, err := answerVerbs(calls, seats, verbs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -65,8 +65,11 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_bounds", History: 1, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
// bounds and test-runs keep keys with their own time to live (§725), as the bus makes them from the manifest's key-ttl.
|
||||
for _, state := range []string{"bounds", TestRunsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The step log, the runner's old logs section (§470): the stream the bus makes for it.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
|
||||
@@ -114,7 +114,11 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
_, err = h.OnStepEnded(bs, m.Subject(), ev)
|
||||
var shared []seat.Shared
|
||||
if bs.Shared != nil {
|
||||
shared = bs.Shared(m.Data())
|
||||
}
|
||||
_, err = h.OnStepEndedShared(bs, m.Subject(), ev, shared)
|
||||
switch {
|
||||
case errors.Is(err, pipeline.ErrRefused):
|
||||
logf("%v", err)
|
||||
@@ -133,11 +137,10 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
return nil
|
||||
}
|
||||
|
||||
// hearAskTaken deletes the bound of each ask a build seat's holder took (§602, §647) and keeps on its step the node that took
|
||||
// it (A-182): the ask is answered, so its five minutes end with no finding.
|
||||
func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue, taken func(ask, node string) error, logf func(string, ...any)) error {
|
||||
// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of
|
||||
// that node's holder, and begins the clock of a test run's ask (§725).
|
||||
func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error {
|
||||
for _, ts := range pipeline.TakenSeats {
|
||||
ts := ts
|
||||
err := hub.Handle(ts.Subject, func(m jetstream.Msg) {
|
||||
var ev nodebuildgo.AskTaken
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
@@ -145,12 +148,6 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue,
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if err := bounds.Delete(ctx, ev.Ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
logf("the bound of ask %s, taken on %s, is not deleted, heard again: %v", ev.Ask, ev.Node, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
// The step keeps the node that took its ask, so a drop of it is asked of that node's holder (A-182).
|
||||
if err := taken(ev.Ask, ev.Node); err != nil {
|
||||
logf("the node %s that took ask %s is not kept on its step, heard again: %v", ev.Node, ev.Ask, err)
|
||||
_ = m.Nak()
|
||||
@@ -165,19 +162,8 @@ func hearAskTaken(ctx context.Context, hub handlers, bounds jetstream.KeyValue,
|
||||
return nil
|
||||
}
|
||||
|
||||
// armBound is the bound of an ask: a key of the state bounds with its own time to live, named for the ask.
|
||||
func armBound(ctx context.Context, bounds jetstream.KeyValue) pipeline.Arm {
|
||||
return func(ask string) error {
|
||||
_, err := bounds.Create(ctx, ask, []byte(`{"ask":"`+ask+`"}`), jetstream.KeyTTL(pipeline.AskBound))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// watchBounds hears the end of each ask's bound, the bus's marker of the key's expiry, and ends the step no holder took (§32,
|
||||
// §535, A-77), and says a drop no holder answered unanswered (A-222): no timer in the process.
|
||||
// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered
|
||||
// unanswered (A-222, A-77): no timer in the process.
|
||||
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
|
||||
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,416 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
||||
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// TestRunsState is the runner's state of its test runs (design 47 G §725), declared in its manifest's old state section.
|
||||
const TestRunsState = "test-runs"
|
||||
|
||||
// testRunsKV is the state test-runs on the bus: its runs by compare-and-set, and its bound keys, each written with its own
|
||||
// time to live (A-77, A-470).
|
||||
type testRunsKV struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (t testRunsKV) Create(key string, r pipeline.Run) error {
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := t.kv.Create(t.ctx, key, raw); errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return pipeline.ErrExists
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t testRunsKV) Get(key string) (pipeline.Run, uint64, error) {
|
||||
e, err := t.kv.Get(t.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return pipeline.Run{}, 0, pipeline.ErrNoRun
|
||||
}
|
||||
if err != nil {
|
||||
return pipeline.Run{}, 0, err
|
||||
}
|
||||
var r pipeline.Run
|
||||
if err := json.Unmarshal(e.Value(), &r); err != nil {
|
||||
return pipeline.Run{}, 0, fmt.Errorf("the test run %s is no test run: %w", key, err)
|
||||
}
|
||||
return r, e.Revision(), nil
|
||||
}
|
||||
|
||||
func (t testRunsKV) Update(key string, r pipeline.Run, revision uint64) (uint64, error) {
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := t.kv.Update(t.ctx, key, raw, revision)
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
|
||||
return 0, pipeline.ErrChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (t testRunsKV) Delete(key string, revision uint64) error {
|
||||
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
|
||||
return pipeline.ErrChanged
|
||||
}
|
||||
if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteBound deletes a bound key, whose value is no run: a key gone is no fault.
|
||||
func (t testRunsKV) deleteBound(key string) error {
|
||||
if err := t.kv.Delete(t.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
|
||||
type boundValue struct {
|
||||
Bound string `json:"bound"`
|
||||
Ask string `json:"ask"`
|
||||
}
|
||||
|
||||
// Bound writes key again holding ask, with ttl as its own time to live, published at the key's latest revision (a delete
|
||||
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
|
||||
// takes its time to live with it, so the expiry is heard only ttl after the last write.
|
||||
func (t testRunsKV) Bound(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
subject := "$KV." + t.kv.Bucket() + "." + key
|
||||
for {
|
||||
var last uint64
|
||||
st, err := t.js.Stream(t.ctx, "KV_"+t.kv.Bucket())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m, err := st.GetLastMsgForSubject(t.ctx, subject)
|
||||
switch {
|
||||
case err == nil:
|
||||
last = m.Sequence
|
||||
case !errors.Is(err, jetstream.ErrMsgNotFound):
|
||||
return 0, err
|
||||
}
|
||||
ack, err := t.js.Publish(t.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
|
||||
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return ack.Sequence, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Bounded reads a bound key: the ask it holds and its revision.
|
||||
func (t testRunsKV) Bounded(key string) (string, uint64, bool, error) {
|
||||
e, err := t.kv.Get(t.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return "", 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", 0, false, err
|
||||
}
|
||||
var v boundValue
|
||||
_ = json.Unmarshal(e.Value(), &v)
|
||||
return v.Ask, e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (t testRunsKV) Unbound(key string) error { return t.deleteBound(key) }
|
||||
|
||||
// UnboundAt deletes a bound key at revision: a later write of it is kept.
|
||||
func (t testRunsKV) UnboundAt(key string, revision uint64) error {
|
||||
err := t.kv.Delete(t.ctx, key, jetstream.LastRevision(revision))
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence") || errors.Is(err, jetstream.ErrKeyNotFound)) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (t testRunsKV) Has(key string) (bool, error) {
|
||||
_, err := t.kv.Get(t.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
func (t testRunsKV) Keys() ([]string, error) {
|
||||
all, err := t.kv.Keys(t.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return all, err
|
||||
}
|
||||
|
||||
// runsKept is the state test-runs as the verb test-runs reads it (A-470i): its keys, and each key's value as written.
|
||||
type runsKept struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (r runsKept) Keys() ([]string, error) {
|
||||
ks, err := r.kv.Keys(r.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return ks, err
|
||||
}
|
||||
|
||||
func (r runsKept) Value(key string) (json.RawMessage, error) {
|
||||
e, err := r.kv.Get(r.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value(), nil
|
||||
}
|
||||
|
||||
// runObjects is the state test-runs' own object store, `test-runs-objects`, made with the state (A-469e), opened when
|
||||
// first used.
|
||||
type runObjects struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
module string
|
||||
mu sync.Mutex
|
||||
store jetstream.ObjectStore
|
||||
}
|
||||
|
||||
func (o *runObjects) open() (jetstream.ObjectStore, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.store != nil {
|
||||
return o.store, nil
|
||||
}
|
||||
s, err := o.js.ObjectStore(o.ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(TestRunsState)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the object store of the state %s is not open: %w", TestRunsState, err)
|
||||
}
|
||||
o.store = s
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (o *runObjects) Put(name string, data []byte) error {
|
||||
s, err := o.open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.PutBytes(o.ctx, name, data)
|
||||
return err
|
||||
}
|
||||
|
||||
func (o *runObjects) Get(name string) ([]byte, error) {
|
||||
s, err := o.open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := s.GetBytes(o.ctx, name)
|
||||
if errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return nil, pipeline.ErrNoRun
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
func (o *runObjects) Delete(name string) error {
|
||||
s, err := o.open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Delete(o.ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runsFill reads test-runs' fill, its key-value stream's bytes (A-469d): judged by the process against its cap setting.
|
||||
func runsFill(ctx context.Context, js jetstream.JetStream, module string, now func() time.Time) func() (seat.Fill, error) {
|
||||
return func() (seat.Fill, error) {
|
||||
st, err := js.Stream(ctx, "KV_"+ownBucket(module, TestRunsState))
|
||||
if err != nil {
|
||||
return seat.Fill{}, err
|
||||
}
|
||||
info, err := st.Info(ctx)
|
||||
if err != nil {
|
||||
return seat.Fill{}, err
|
||||
}
|
||||
at := now().UTC()
|
||||
return seat.Fill{Store: TestRunsState, Bytes: int64(info.State.Bytes), Cap: info.Config.MaxBytes, At: &at}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// The files the manifest writes each setting of the test runs into, `${config:<key>}`, named by the environment (A-469x).
|
||||
const (
|
||||
envRunsCap = "MESH_RUNNER_TEST_RUNS_CAP_FILE"
|
||||
envRunsKept = "MESH_RUNNER_TEST_RUNS_KEPT_FILE"
|
||||
envStepBound = "MESH_RUNNER_TEST_STEP_BOUND_FILE"
|
||||
envUntakenBound = "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE"
|
||||
)
|
||||
|
||||
// maxRunsKept is test-runs-kept's maximum (A-470a): the pipeline naming a run's key is kept at least that long.
|
||||
const maxRunsKept = 168 * time.Hour
|
||||
|
||||
// runSettings reads the settings of the test runs at the start (§725; A-469x): test-runs-cap-mib a whole number of MiB,
|
||||
// test-runs-kept, test-step-bound and run-untaken-bound Go durations. A setting whose file is missing or empty is its
|
||||
// default; a content that is neither is an error naming the file.
|
||||
func runSettings(getenv func(string) string) (pipeline.RunSettings, error) {
|
||||
s := pipeline.DefaultRunSettings
|
||||
if v, err := settingText(getenv(envRunsCap)); err != nil {
|
||||
return s, err
|
||||
} else if v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
return s, fmt.Errorf("%s holds %q, which is not a positive whole number", getenv(envRunsCap), v)
|
||||
}
|
||||
s.CapBytes = int64(n) << 20
|
||||
}
|
||||
for _, d := range []struct {
|
||||
env string
|
||||
to *time.Duration
|
||||
}{{envRunsKept, &s.Kept}, {envStepBound, &s.StepBound}, {envUntakenBound, &s.Untaken}} {
|
||||
v, err := settingText(getenv(d.env))
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
t, err := time.ParseDuration(v)
|
||||
if err != nil || t <= 0 {
|
||||
return s, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(d.env), v)
|
||||
}
|
||||
*d.to = t
|
||||
}
|
||||
if s.Kept > maxRunsKept {
|
||||
return s, fmt.Errorf("%s holds %s, past test-runs-kept's maximum %s (A-470a)", getenv(envRunsKept), s.Kept, maxRunsKept)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// settingText is one setting file's content, trimmed: empty for no path, no file or an empty one.
|
||||
func settingText(path string) (string, error) {
|
||||
if path == "" {
|
||||
return "", nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(raw)), nil
|
||||
}
|
||||
|
||||
// progressSubjects hear every build seat's step-progress (§725), from their contracts.
|
||||
var progressSubjects = []string{nodebuildgo.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll}
|
||||
|
||||
// hearStepProgress begins again the clock of the test run whose ask a build seat's holder says progress of (§725, A-466p).
|
||||
func hearStepProgress(hub handlers, progress func(ask string) error, logf func(string, ...any)) error {
|
||||
for _, subject := range progressSubjects {
|
||||
err := hub.Handle(subject, func(m jetstream.Msg) {
|
||||
var ev nodebuildgo.StepProgress
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
logf("%s is no step-progress: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if err := progress(ev.Ask); err != nil {
|
||||
logf("the clock of the test run of ask %s is not begun again, heard again: %v", ev.Ask, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// watchRunBounds hears the expiry of each test run's bound key, the bus's marker of the key's end (§725): no timer in the
|
||||
// process.
|
||||
func watchRunBounds(ctx context.Context, kv jetstream.KeyValue, expired func(key string) error, logf func(string, ...any)) error {
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = w.Stop()
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePurge || !pipeline.IsRunBound(e.Key()) {
|
||||
continue
|
||||
}
|
||||
if err := expired(e.Key()); err != nil {
|
||||
logf("the test run of the bound %s that ended is not ended: %v", e.Key(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// openTestRuns opens the state test-runs and wires it into the starter, the hearer and the ager (§725).
|
||||
func openTestRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, getenv func(string) string,
|
||||
starter *pipeline.Starter, hearer *pipeline.Hearer, drop pipeline.Drop, logf func(string, ...any)) (jetstream.KeyValue, *runObjects, error) {
|
||||
settings, err := runSettings(getenv)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
kv, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, TestRunsState), "share a test run")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
runs := testRunsKV{ctx: ctx, js: js, kv: kv}
|
||||
objects := &runObjects{ctx: ctx, js: js, module: cred.Module}
|
||||
starter.TestRuns, starter.Clock, starter.Runs = runs, runs, settings
|
||||
starter.RunsFill = runsFill(ctx, js, cred.Module, time.Now)
|
||||
starter.EndReaders = hearer.EndReaders
|
||||
hearer.TestRuns, hearer.Clock, hearer.Runs, hearer.DropRun = runs, runs, settings, drop
|
||||
hearer.RunOutputs = objects
|
||||
if hearer.Ager != nil {
|
||||
hearer.Ager.Runs, hearer.Ager.RunOutputs, hearer.Ager.RunsKept = runs, objects.Delete, settings.Kept
|
||||
}
|
||||
logf("test runs: cap %d bytes, kept %s, step bound %s, untaken bound %s", settings.CapBytes, settings.Kept, settings.StepBound, settings.Untaken)
|
||||
return kv, objects, nil
|
||||
}
|
||||
+579
@@ -0,0 +1,579 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// Test 109 (design 47 G §733 *Whole (runner…)*, §724 to §728, §535 and §647 as A-466y and A-466z amend them; A-468f: in
|
||||
// mesh-delivery with a fixture build seat holder; A-470f: no reader-only refusal). The runner whole against the real bus: a
|
||||
// module whose program and tools bundle test one directory asks one unit and one integration run, both artifacts' steps
|
||||
// ending with its result and shared; a failing test fails once and is counted once; a rerun of one artifact's failed step
|
||||
// that passes moves the other's, the pipeline ending again, and an accepted step does not move; a runner restarted during a
|
||||
// run takes it up by its ask, its clock holding that ask (A-470n); a test run waiting with a holder present is not failed;
|
||||
// a replaced reader leaves the ask to the other readers; a run that keeps saying its progress past its bound is not ended,
|
||||
// and a taken ask silent for its bound ends could not run with its readers, its ask dropped; a second revision with the
|
||||
// same digests asks its own runs; a seat the seat-holders view names no holder for ends its pipeline failed at once, its
|
||||
// other steps not asked (A-470g, A-470h); the process answers test-runs with its fill against test-runs-cap-mib and
|
||||
// test-runs-drop deletes the key and its output object (A-470i); at the cap a step ends could not run, naming the fill.
|
||||
// The step bound is two seconds here, as the setting test-step-bound names it, where the mesh's is 30 minutes.
|
||||
func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
const module = "mesh-delivery-pipeline"
|
||||
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, state := range []string{"bounds", TestRunsState} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// test-runs' own object store, made with the state (A-469e).
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", "mesh.stream.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "n1_" + module, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A durable consumer of the runner's step-ended in EVENTS, so an end said again is read as the stream keeps it (A-470q).
|
||||
stepEnds, err := events.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: "t109-step-ended",
|
||||
FilterSubject: "mesh.seat." + seat.Seat + ".event." + seat.EventStepEnded + ".>", AckPolicy: jetstream.AckExplicitPolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASKS", Subjects: []string{"mesh.seat.*.accept.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The seat-holders view of the seat mesh-module-assigner's claimant (§647, A-466z), served as its owner does: node-build-go
|
||||
// has a holder, node-build-node none.
|
||||
holders := viewstest.Serve(t, ctx, js, nc, "assigner", "seat-holders", "assigner.seat-holders", "n1_"+module)
|
||||
if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The registrations: app tests its program and tools bundle from one directory, its unit step required and its
|
||||
// integration step optional; web is built by node-build-node.
|
||||
policy := catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required", Integration: "optional", Composition: "bypass", Review: "bypass", Approval: "bypass"}}
|
||||
regs := []catalog.Registration{{Module: "app", Branches: []string{"main"}, Origin: "o/app", Policy: policy},
|
||||
{Module: "web", Branches: []string{"main"}, Origin: "o/web", Policy: policy}}
|
||||
answer := func(subject string, body any) {
|
||||
if _, err := nc.Subscribe(subject, func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": body})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
answer("mesh.seat.mesh-catalog.call.registrations.*", catalog.RegistrationsAnswer{Registrations: regs})
|
||||
answer("mesh.seat.mesh-hand-acts.call.hand-act.*", map[string]any{})
|
||||
entries := []catalog.CatalogEntry{
|
||||
{Module: "app", Commit: "c1", Fp: "src2:aa", Path: "modules/app", Repository: "o/app",
|
||||
Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
{Module: "web", Commit: "w1", Fp: "src2:ww", Path: "modules/web", Repository: "o/web", Artifacts: []catalog.Artifact{{Ref: "web:program", Language: "node"}}},
|
||||
{Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
{Module: "mesh-build-node", Acts: []catalog.ActingRule{{Seat: "node-build-node", Step: "build", On: "node"}}},
|
||||
{Module: "assigner", Manifest: map[string]any{"module": "assigner", "version": "0.1", "claims": []any{map[string]any{"name": seatlist.HolderSeat, "scope": "mesh"}}}},
|
||||
}
|
||||
var revision atomic.Int64
|
||||
revision.Store(1)
|
||||
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call.revision.*", func(m *nats.Msg) {
|
||||
r := revision.Load()
|
||||
raw, _ := json.Marshal(map[string]any{"result": catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: r, Kind: "trunk", Branch: "main", Source: 2 + r}, Entries: entries}})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
def, err := os.ReadFile("../../../mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.Subscribe("mesh.seat.mesh-catalog.call."+seatlist.Verb+".*", func(m *nats.Msg) {
|
||||
var in struct {
|
||||
Seat string `json:"seat"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
entry := seatlist.Entry{Seat: seat.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:runner"}
|
||||
var result any = seatlist.Listing{Revision: 1, Seats: []seatlist.Entry{entry}}
|
||||
if in.Seat != "" {
|
||||
entry.Definition = def
|
||||
result = seatlist.Answer{Seat: in.Seat, Found: in.Seat == seat.Seat, Entry: &entry, Revision: 1}
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"result": result})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Every pipeline-ended said, and every drop asked of a build seat.
|
||||
var mu sync.Mutex
|
||||
var endings []seat.PipelineEnded
|
||||
var drops []string
|
||||
if _, err := nc.Subscribe("mesh.seat.mesh-pipeline-runner.event.pipeline-ended.>", func(m *nats.Msg) {
|
||||
var e seat.PipelineEnded
|
||||
if json.Unmarshal(m.Data, &e) == nil {
|
||||
mu.Lock()
|
||||
endings = append(endings, e)
|
||||
mu.Unlock()
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.Subscribe("mesh.seat.*.call.drop.>", func(m *nats.Msg) {
|
||||
mu.Lock()
|
||||
drops = append(drops, string(m.Data))
|
||||
mu.Unlock()
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{}})
|
||||
_ = m.Respond(raw)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ended := func(key string) []seat.PipelineEnded {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
var out []seat.PipelineEnded
|
||||
for _, e := range endings {
|
||||
if e.Pipeline == key {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds.
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{envRunsCap: "1", envStepBound: "2s"}
|
||||
env := map[string]string{}
|
||||
for name, content := range files {
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env[name] = path
|
||||
}
|
||||
broker := filepath.Join(dir, "broker")
|
||||
raw, _ := json.Marshal(process.Account{URL: url, Node: "n1", Module: module, User: "runner", Password: "x"})
|
||||
if err := os.WriteFile(broker, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env[process.BrokerFileEnv] = broker
|
||||
logs := &syncBuffer{}
|
||||
start := func() func() {
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, logs) }()
|
||||
return func() { stop(); <-done }
|
||||
}
|
||||
stop := start()
|
||||
defer func() { stop() }()
|
||||
|
||||
pipelines, err := js.KeyValue(ctx, module+"_pipelines")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runsKV, err := js.KeyValue(ctx, module+"_"+TestRunsState)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstViews, err := js.KeyValue(ctx, module+"_"+views.StatusState)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The seat-holders view's first view is read before any pipeline is placed, so a seat is judged from it (A-466y).
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e, err := firstViews.Get(ctx, views.StatusKey("assigner", "seat-holders"))
|
||||
var s views.Status
|
||||
return err == nil && json.Unmarshal(e.Value(), &s) == nil && !s.Answered.IsZero()
|
||||
})
|
||||
|
||||
takeIn := func(module, commit, fp string, source, rev int64) {
|
||||
taken, _ := json.Marshal(catalog.TakeInTaken{Key: "m." + commit, Kind: "taken", Branch: "main", Source: source, Revision: rev,
|
||||
Entries: []catalog.TakeInEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
|
||||
if _, err := js.Publish(ctx, "mesh.mod.mesh-delivery-indexer.event.take-in.taken", taken); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
placed := func(key string) seat.Pipeline {
|
||||
var p seat.Pipeline
|
||||
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && len(p.Steps) > 1 })
|
||||
return p
|
||||
}
|
||||
read := func(key string) seat.Pipeline {
|
||||
var p seat.Pipeline
|
||||
readPipeline(ctx, pipelines, key, &p)
|
||||
return p
|
||||
}
|
||||
step := func(p seat.Pipeline, artifact, name string) seat.StepResult {
|
||||
for _, s := range p.Steps {
|
||||
if s.Artifact == artifact && s.Step == name {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return seat.StepResult{}
|
||||
}
|
||||
runOf := func(key string) (pipeline.Run, bool) {
|
||||
e, err := runsKV.Get(ctx, key)
|
||||
var r pipeline.Run
|
||||
return r, err == nil && json.Unmarshal(e.Value(), &r) == nil
|
||||
}
|
||||
runKeys := func(candidate string) []string {
|
||||
ks, _ := runsKV.Keys(ctx)
|
||||
var out []string
|
||||
for _, k := range ks {
|
||||
if !pipeline.IsRunBound(k) && strings.HasPrefix(k, candidate+".") {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
runOfStep := func(key, artifact, name string) (string, pipeline.Run) {
|
||||
var run string
|
||||
var r pipeline.Run
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
s := step(read(key), artifact, name)
|
||||
if len(s.Shared) == 0 {
|
||||
return false
|
||||
}
|
||||
run = s.Shared[0].Run
|
||||
got, ok := runOf(run)
|
||||
r = got
|
||||
return ok && r.Ask != ""
|
||||
})
|
||||
return run, r
|
||||
}
|
||||
publish := func(subject string, body any) {
|
||||
raw, _ := json.Marshal(body)
|
||||
if _, err := js.Publish(ctx, subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
taken := func(ask string) {
|
||||
publish(nodebuildgo.SubjectAskTaken("n1"), nodebuildgo.AskTaken{Ask: ask, Asker: module, Node: "n1", At: time.Now()})
|
||||
}
|
||||
endAsk := func(key, ask, name, outcome string, tests *nodebuildgo.TestCounts) {
|
||||
now := time.Now()
|
||||
publish(nodebuildgo.SubjectStepEnded("n1"), nodebuildgo.StepEnded{ID: ask, Ask: ask, Asker: module, Pipeline: key, Module: "app", Commit: "c1",
|
||||
Fingerprint: "src2:aa", Step: name, Node: "n1", Outcome: outcome, Tests: tests, At: now, Ended: &now})
|
||||
}
|
||||
passBuild := func(key, module, commit, fp string) {
|
||||
ask := step(read(key), "", seat.StepResultStepBuild).Ask
|
||||
publish("mesh.seat.mesh-builds.event.step-ended.n1", builds.StepEnded{ID: ask, Pipeline: key, Step: "build", Node: "n1", Outcome: "passed",
|
||||
Entries: []builds.BuildEntry{{Module: module, Commit: commit, Fingerprint: fp}}})
|
||||
}
|
||||
call := func(verb string, in, out any) string {
|
||||
var reply struct {
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
m, err := nc.Request("mesh.seat.mesh-pipeline-runner.call."+verb+".agent~dev", mustJSON(in), 2*time.Second)
|
||||
reply.Error = ""
|
||||
return err == nil && json.Unmarshal(m.Data, &reply) == nil && !strings.Contains(reply.Error, "not yet read")
|
||||
})
|
||||
if out != nil && reply.Error == "" {
|
||||
_ = json.Unmarshal(reply.Result, out)
|
||||
}
|
||||
return reply.Error
|
||||
}
|
||||
|
||||
// One unit run and one integration run for both artifacts (§724), each read by both, waiting for a build node until its
|
||||
// ask is taken (A-469b), its holder present and nothing failing it (A-466y).
|
||||
const p3 = "main.3.app"
|
||||
takeIn("app", "c1", "src2:aa", 3, 1)
|
||||
passBuild(p3, "app", "c1", "src2:aa")
|
||||
unitRun, unit := runOfStep(p3, "app:program", "unit")
|
||||
integRun, integ := runOfStep(p3, "app:program", "integration")
|
||||
if got := runKeys("main.1"); len(got) != 2 || len(unit.Readers) != 2 || len(integ.Readers) != 2 {
|
||||
t.Fatalf("runs %v, unit readers %+v, integration readers %+v; want one run per level read by both artifacts", got, unit.Readers, integ.Readers)
|
||||
}
|
||||
if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask {
|
||||
t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s)
|
||||
}
|
||||
// Its wait has no bound (§647 as A-466y amends it): the state bounds holds no key for a queued ask, only drops' keys.
|
||||
boundsKV, err := js.KeyValue(ctx, module+"_bounds")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ks, err := boundsKV.Keys(ctx); err != nil && err != jetstream.ErrNoKeysFound {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
for _, k := range ks {
|
||||
if !strings.HasPrefix(k, pipeline.DropBoundPrefix) {
|
||||
t.Errorf("the state bounds holds %s after the asks were put; want no bound of a queued ask", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A runner restarted during a run takes it up again by its ask (§725): the run's end, heard after the restart, ends both.
|
||||
taken(unit.Ask)
|
||||
waitFor(t, ctx, logs, func() bool { r, _ := runOf(unitRun); return r.Node == "n1" })
|
||||
// Its clock holds the run's ask (A-470n).
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun)
|
||||
return err == nil && strings.Contains(string(e.Value()), `"ask":"`+unit.Ask+`"`)
|
||||
})
|
||||
stop()
|
||||
stop = start()
|
||||
// It keeps saying its progress past its bound and is not ended (§725, §720).
|
||||
// Each progress is heard before the next is said: the run's clock is written again (its bound key's revision rises).
|
||||
boundRev := func() uint64 {
|
||||
e, err := runsKV.Get(ctx, pipeline.BoundPrefix+unitRun)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return e.Revision()
|
||||
}
|
||||
for at := time.Now(); time.Since(at) < 3*time.Second; {
|
||||
before := boundRev()
|
||||
publish(nodebuildgo.SubjectStepProgress("n1"), nodebuildgo.StepProgress{Ask: unit.Ask, Asker: module, Phase: "run", At: time.Now()})
|
||||
waitFor(t, ctx, logs, func() bool { return boundRev() > before })
|
||||
}
|
||||
if r, _ := runOf(unitRun); r.Ended != nil {
|
||||
t.Fatalf("a run saying its progress past its bound ended: %+v", r)
|
||||
}
|
||||
// A failing test fails once and is counted once (§726).
|
||||
endAsk(p3, unit.Ask, "unit", "failed", &nodebuildgo.TestCounts{Unit: 5})
|
||||
taken(integ.Ask)
|
||||
endAsk(p3, integ.Ask, "integration", "failed", &nodebuildgo.TestCounts{Integration: 2})
|
||||
waitFor(t, ctx, logs, func() bool { return len(ended(p3)) == 1 })
|
||||
p := read(p3)
|
||||
for _, a := range []string{"app:program", "app:tools"} {
|
||||
if s := step(p, a, "unit"); s.Outcome != seat.StepResultOutcomeFailed || len(s.Shared) == 0 || s.Shared[0].Run != unitRun {
|
||||
t.Errorf("%s's unit step = %+v; want failed with the run in shared", a, s)
|
||||
}
|
||||
}
|
||||
for _, k := range []string{unitRun, integRun} {
|
||||
if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomeFailed {
|
||||
t.Errorf("the run %s at its end = %+v; want it ended failed", k, r)
|
||||
}
|
||||
}
|
||||
if e := ended(p3)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Tests == nil || e.Tests.Unit != 5 || e.Tests.Integration != 2 {
|
||||
t.Fatalf("pipeline-ended = %+v; want failed, the tests counted once", e)
|
||||
}
|
||||
|
||||
// An accepted step does not move; a rerun of the program's unit step that passes moves the tools' to passed, and the
|
||||
// pipeline says pipeline-ended again (§727, A-470d).
|
||||
if e := call(seat.VerbAccept, seat.AcceptInput{Pipeline: p3, Step: "integration", Artifact: "app:program", Why: "flaky"}, nil); e != "" {
|
||||
t.Fatalf("accept: %s", e)
|
||||
}
|
||||
if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "unit", Artifact: "app:program", Why: "fixed"}, nil); e != "" {
|
||||
t.Fatalf("rerun: %s", e)
|
||||
}
|
||||
var again pipeline.Run
|
||||
waitFor(t, ctx, logs, func() bool { r, ok := runOf(unitRun); again = r; return ok && r.Ended == nil && r.Ask != unit.Ask })
|
||||
taken(again.Ask)
|
||||
endAsk(p3, again.Ask, "unit", "passed", &nodebuildgo.TestCounts{Unit: 5})
|
||||
if e := call(seat.VerbRerun, seat.RerunInput{Pipeline: p3, Step: "integration", Artifact: "app:tools", Why: "fixed"}, nil); e != "" {
|
||||
t.Fatalf("rerun: %s", e)
|
||||
}
|
||||
var integAgain pipeline.Run
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
r, ok := runOf(integRun)
|
||||
integAgain = r
|
||||
return ok && r.Ended == nil && r.Ask != integ.Ask
|
||||
})
|
||||
taken(integAgain.Ask)
|
||||
endAsk(p3, integAgain.Ask, "integration", "passed", nil)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e := ended(p3)
|
||||
return len(e) >= 2 && e[len(e)-1].Outcome == seat.PipelineEndedOutcomePassed
|
||||
})
|
||||
p = read(p3)
|
||||
if s := step(p, "app:tools", "unit"); s.Outcome != seat.StepResultOutcomePassed || !s.Rerun {
|
||||
t.Errorf("the tools' unit step = %+v; want moved to passed, marked a rerun", s)
|
||||
}
|
||||
// The tools' unit step's ends, read from the stream: its first, failed, and its end said again by the move, passed.
|
||||
var toolEnds []string
|
||||
for {
|
||||
batch, err := stepEnds.FetchNoWait(100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := 0
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
var e seat.StepEvent
|
||||
if json.Unmarshal(m.Data(), &e) == nil && e.Pipeline == p3 && e.Result.Artifact == "app:tools" && e.Result.Step == "unit" {
|
||||
toolEnds = append(toolEnds, e.Result.Outcome)
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(toolEnds) < 2 || toolEnds[0] != seat.StepResultOutcomeFailed || toolEnds[len(toolEnds)-1] != seat.StepResultOutcomePassed {
|
||||
t.Errorf("the tools' unit step-ended in EVENTS = %v; want its first end failed and the end said again passed, both kept", toolEnds)
|
||||
}
|
||||
if s := step(p, "app:program", "integration"); s.Outcome != seat.StepResultOutcomeAccepted {
|
||||
t.Errorf("the accepted step = %+v; want it accepted still", s)
|
||||
}
|
||||
for _, k := range []string{unitRun, integRun} {
|
||||
if r, _ := runOf(k); r.Ended == nil || r.Outcome != seat.StepResultOutcomePassed {
|
||||
t.Errorf("the run %s after its rerun = %+v; want it ended passed", k, r)
|
||||
}
|
||||
}
|
||||
if e := ended(p3); e[len(e)-1].EndCount <= e[0].EndCount {
|
||||
t.Errorf("pipeline-ended counts %d then %d; want a higher end count", e[0].EndCount, e[len(e)-1].EndCount)
|
||||
}
|
||||
|
||||
// A second revision with the same digests asks its own runs, so a failed test runs again (§724).
|
||||
revision.Store(2)
|
||||
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 2, Kind: "trunk", Branch: "main", Source: 4})
|
||||
const p4, p5 = "main.4.app", "main.5.app"
|
||||
takeIn("app", "c1", "src2:aa", 4, 2)
|
||||
placed(p4)
|
||||
run4, r4 := runOfStep(p4, "app:program", "integration")
|
||||
if !strings.HasPrefix(run4, "main.2.") || r4.Ask == integ.Ask {
|
||||
t.Fatalf("revision 2's integration run %s (%s); want its own", run4, r4.Ask)
|
||||
}
|
||||
// A replaced reader leaves the ask to the other readers (§726): a second pipeline of the same candidate and digest reads
|
||||
// the run, the first is cancelled and leaves it, and the run's ask is not dropped.
|
||||
takeIn("app", "c1", "src2:aa", 5, 2)
|
||||
placed(p5)
|
||||
waitFor(t, ctx, logs, func() bool { r, _ := runOf(run4); return len(r.Readers) == 4 })
|
||||
if e := call(seat.VerbCancel, seat.CancelInput{Pipeline: p4, Why: "replaced"}, nil); e != "" {
|
||||
t.Fatalf("cancel: %s", e)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
r, _ := runOf(run4)
|
||||
return len(r.Readers) == 2 && r.Readers[0].Pipeline == p5 && r.Readers[1].Pipeline == p5
|
||||
})
|
||||
if r, _ := runOf(run4); r.Ended != nil || r.Ask != r4.Ask {
|
||||
t.Fatalf("the run of the replaced readers = %+v; want it running under its ask", r)
|
||||
}
|
||||
mu.Lock()
|
||||
dropped := append([]string(nil), drops...)
|
||||
mu.Unlock()
|
||||
if len(dropped) != 0 {
|
||||
t.Fatalf("drops asked %v; want none while a reader is left\n%s", dropped, logs.String())
|
||||
}
|
||||
// A taken ask silent for its bound ends could not run with its readers, its ask dropped (§725, §720).
|
||||
taken(r4.Ask)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
return step(read(p5), "app:tools", "integration").Outcome == seat.StepResultOutcomeCouldNotRun &&
|
||||
step(read(p5), "app:program", "integration").Outcome == seat.StepResultOutcomeCouldNotRun
|
||||
})
|
||||
if r, _ := runOf(run4); r.Ended == nil || r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "test-step-bound") {
|
||||
t.Errorf("the silent run = %+v; want it ended could not run, naming its bound", r)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(drops) == 1 && strings.Contains(drops[0], r4.Ask)
|
||||
})
|
||||
|
||||
// A seat the seat-holders view names no holder for ends the pipeline failed at once, naming it, its other steps not
|
||||
// asked (§535, §647; A-470g, A-470h).
|
||||
const web = "main.5.web"
|
||||
takeIn("web", "w1", "src2:ww", 5, 2)
|
||||
waitFor(t, ctx, logs, func() bool { return len(ended(web)) == 1 })
|
||||
if e := ended(web)[0]; e.Outcome != seat.PipelineEndedOutcomeFailed || e.Why != "the seat node-build-node has no holder: the seat-holders view names none" {
|
||||
t.Fatalf("pipeline-ended = %+v; want failed at once naming the seat", e)
|
||||
}
|
||||
// The steps of the seat with no holder end could not run, a bypassed step stays bypassed, every other ends not asked,
|
||||
// and none is asked (A-470h).
|
||||
for _, s := range read(web).Steps {
|
||||
want := seat.StepResultOutcomeNotAsked
|
||||
switch {
|
||||
case s.Status == seat.StepResultStatusBypass:
|
||||
want = seat.StepResultOutcomeBypassed
|
||||
case s.Seat == "node-build-node":
|
||||
want = seat.StepResultOutcomeCouldNotRun
|
||||
}
|
||||
if s.Outcome != want || s.Ask != "" {
|
||||
t.Errorf("the step %s %s of the pipeline with no holder = %s (ask %q); want %s", s.Artifact, s.Step, s.Outcome, s.Ask, want)
|
||||
}
|
||||
}
|
||||
|
||||
// test-runs is answered by the process, its fill against test-runs-cap-mib (A-470i), each run's packages from its output
|
||||
// object; test-runs-drop deletes the key and its object (§728).
|
||||
var runs seat.TestRunsAnswer
|
||||
if e := call(seat.VerbTestRuns, seat.TestRunsInput{Candidate: "main.1"}, &runs); e != "" || len(runs.Runs) != 2 || runs.Fill.Cap != 1<<20 || runs.Fill.Store != TestRunsState {
|
||||
t.Fatalf("test-runs = %+v (%s); want main.1's two runs, the fill against 1 MiB", runs, e)
|
||||
}
|
||||
objects, err := js.ObjectStore(ctx, catalogue.BusName(module, catalogue.StateObjectsName(TestRunsState)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := objects.GetInfo(ctx, unitRun); err != nil {
|
||||
t.Fatalf("the unit run's output object: %v", err)
|
||||
}
|
||||
var drop seat.TestRunsDropAnswer
|
||||
if e := call(seat.VerbTestRunsDrop, seat.TestRunsDropInput{Key: unitRun, Why: "test 109"}, &drop); e != "" || drop.Run.Key != unitRun {
|
||||
t.Fatalf("test-runs-drop = %+v (%s)", drop, e)
|
||||
}
|
||||
if _, ok := runOf(unitRun); ok {
|
||||
t.Error("the dropped run's key is kept")
|
||||
}
|
||||
if _, err := objects.GetInfo(ctx, unitRun); err == nil {
|
||||
t.Error("the dropped run's output object is kept")
|
||||
}
|
||||
|
||||
// At the cap a step ends could not run, naming the fill (§725, A-469d).
|
||||
pad := make([]byte, 600<<10)
|
||||
for _, k := range []string{"pad.a", "pad.b"} {
|
||||
if _, err := runsKV.Put(ctx, k, pad); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
revision.Store(3)
|
||||
publish(catalog.SubjectCatalogRevision("main"), catalog.CatalogRevisionEvent{V: 1, Revision: 3, Kind: "trunk", Branch: "main", Source: 5})
|
||||
const p6 = "main.6.app"
|
||||
takeIn("app", "c1", "src2:aa", 6, 3)
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
s := step(read(p6), "app:program", "unit")
|
||||
return s.Outcome == seat.StepResultOutcomeCouldNotRun && strings.Contains(s.Why, "test-runs-cap-mib")
|
||||
})
|
||||
}
|
||||
|
||||
func mustJSON(v any) []byte {
|
||||
raw, _ := json.Marshal(v)
|
||||
return raw
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The test runs' settings reach the process through the files the manifest writes (design 47 G §725; A-469x, A-470a): the
|
||||
// cap a whole number of MiB, the times Go durations, a missing file its default, a content that is neither, or a kept time
|
||||
// past 168h, an error naming the file.
|
||||
func TestTheTestRunsSettingsAreReadFromTheirFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, content string) string {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
rows := []struct {
|
||||
name string
|
||||
files map[string]string
|
||||
ok bool
|
||||
check func(t *testing.T, cap int64, kept, step, untaken time.Duration)
|
||||
}{
|
||||
{"defaults", nil, true, func(t *testing.T, cap int64, kept, step, untaken time.Duration) {
|
||||
if cap != 16<<20 || kept != 168*time.Hour || step != 30*time.Minute || untaken != 24*time.Hour {
|
||||
t.Errorf("defaults = %d %s %s %s", cap, kept, step, untaken)
|
||||
}
|
||||
}},
|
||||
{"set", map[string]string{envRunsCap: "8\n", envRunsKept: "48h", envStepBound: "2s", envUntakenBound: "1m"}, true,
|
||||
func(t *testing.T, cap int64, kept, step, untaken time.Duration) {
|
||||
if cap != 8<<20 || kept != 48*time.Hour || step != 2*time.Second || untaken != time.Minute {
|
||||
t.Errorf("settings = %d %s %s %s", cap, kept, step, untaken)
|
||||
}
|
||||
}},
|
||||
{"a cap that is no whole number", map[string]string{envRunsCap: "8.5"}, false, nil},
|
||||
{"a time that is no duration", map[string]string{envStepBound: "30"}, false, nil},
|
||||
{"kept past its maximum", map[string]string{envRunsKept: "169h"}, false, nil},
|
||||
}
|
||||
for _, r := range rows {
|
||||
t.Run(r.name, func(t *testing.T) {
|
||||
env := map[string]string{}
|
||||
for k, v := range r.files {
|
||||
env[k] = write(r.name+k, v)
|
||||
}
|
||||
s, err := runSettings(func(k string) string { return env[k] })
|
||||
if (err == nil) != r.ok {
|
||||
t.Fatalf("runSettings = %+v, %v", s, err)
|
||||
}
|
||||
if r.check != nil {
|
||||
r.check(t, s.CapBytes, s.Kept, s.StepBound, s.Untaken)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
@@ -41,7 +38,8 @@ func answerVerbs(conn *process.Conn, seats *seatlist.Part, v pipeline.Verbs) (fu
|
||||
return conn.AnswerSeat(seat.Seat, seatlist.Definition(seats.Reader, seat.Seat), writeHandlers(v))
|
||||
}
|
||||
|
||||
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again.
|
||||
// writeHandlers are the seat's write verbs the process answers: run, review, rerun, accept, cancel, start, and ask-again; and
|
||||
// the test runs' verbs test-runs and test-runs-drop (A-469g, A-470i).
|
||||
func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
|
||||
return map[string]process.SeatHandler{
|
||||
seat.VerbRun: verbOf(func(c string, in seat.RunInput) (any, error) { return v.Run(c, in) }),
|
||||
@@ -51,24 +49,17 @@ func writeHandlers(v pipeline.Verbs) map[string]process.SeatHandler {
|
||||
seat.VerbCancel: verbOf(func(c string, in seat.CancelInput) (any, error) { return v.Cancel(c, in) }),
|
||||
seat.VerbStart: verbOf(func(c string, in seat.StartInput) (any, error) { return v.Start(c, in) }),
|
||||
seat.VerbAskAgain: verbOf(func(c string, in seat.AskAgainInput) (any, error) { return v.AskAgain(c, in) }),
|
||||
}
|
||||
}
|
||||
|
||||
// dropBound deletes the bound of an ask a cancelled pipeline no longer waits on.
|
||||
func dropBound(ctx context.Context, bounds jetstream.KeyValue) func(string) error {
|
||||
return func(ask string) error {
|
||||
if err := bounds.Delete(ctx, ask); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
// §728, A-469g: a test run's drop is answered by the process, the only writer of pipelines and test-runs.
|
||||
seat.VerbTestRunsDrop: verbOf(func(c string, in seat.TestRunsDropInput) (any, error) { return v.TestRunsDrop(c, in) }),
|
||||
// A-470i: test-runs too, since the process holds test-runs-cap-mib, the cap its fill is said against.
|
||||
seat.VerbTestRuns: verbOf(func(_ string, in seat.TestRunsInput) (any, error) { return v.Runs.TestRuns(in) }),
|
||||
}
|
||||
}
|
||||
|
||||
// writeVerbs are the verbs over the hearer and the starter, their hand acts recorded through the seat mesh-hand-acts.
|
||||
func writeVerbs(ctx context.Context, h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool),
|
||||
calls *process.Conn, bounds jetstream.KeyValue) pipeline.Verbs {
|
||||
func writeVerbs(h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipeline.Half, bool), calls *process.Conn) pipeline.Verbs {
|
||||
candidate := func(group, branch string, revision int64) (pipeline.Half, error) {
|
||||
return readCandidate(calls.CallKept, catalog.CatalogRevisionEvent{Kind: "candidate", Group: group, Branch: branch, Revision: revision})
|
||||
}
|
||||
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}, Drop: dropBound(ctx, bounds)}
|
||||
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ func TestEveryWriteVerbIsServed(t *testing.T) {
|
||||
got = append(got, v)
|
||||
}
|
||||
slices.Sort(got)
|
||||
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun}
|
||||
want := []string{seat.VerbAccept, seat.VerbAskAgain, seat.VerbCancel, seat.VerbRerun, seat.VerbReview, seat.VerbStart, seat.VerbRun, seat.VerbTestRuns, seat.VerbTestRunsDrop}
|
||||
slices.Sort(want)
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("the process answers %v, want %v", got, want)
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
@@ -26,6 +28,70 @@ type runnerViews struct {
|
||||
// Findings is the runner's held finding writer: it raises the finding pipeline-failed (§33) and says the runner's own
|
||||
// findings (A-206).
|
||||
Findings *findingsWriter
|
||||
// Holders follows the seat-holders view of the seat mesh-module-assigner's claimant (§535, §647; A-466z).
|
||||
Holders *holderView
|
||||
}
|
||||
|
||||
// holderView follows the view seat-holders of the module the catalog halves name as the claimant of the seat
|
||||
// mesh-module-assigner (seatlist.HolderSeat), as the runner's manifest consumes it, `seat:mesh-module-assigner.views.seat-holders`
|
||||
// (design 47 G §535, §647 as A-466z amends them; §717): a claimant named anew drops the view followed and follows the new one's.
|
||||
// Nothing is asked on a timer: it is read again whenever a branch's half is held or advances.
|
||||
type holderView struct {
|
||||
reader *views.ProcessReader
|
||||
catalog *pipeline.Views
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
owner string
|
||||
pic *views.Picture
|
||||
}
|
||||
|
||||
// refresh follows the claimant's seat-holders view when the claimant is another than the one followed.
|
||||
func (h *holderView) refresh() {
|
||||
var halves []pipeline.Half
|
||||
for _, b := range h.catalog.Branches() {
|
||||
if half, ok := h.catalog.View(b); ok {
|
||||
halves = append(halves, half)
|
||||
}
|
||||
}
|
||||
claimant, ok := pipeline.ClaimantOf(halves, seatlist.HolderSeat)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if !ok || claimant == h.owner {
|
||||
return
|
||||
}
|
||||
if h.owner != "" {
|
||||
if err := h.reader.Drop(h.owner, seatHoldersState); err != nil {
|
||||
h.logf("the seat-holders view of %s is not dropped: %v", h.owner, err)
|
||||
}
|
||||
h.owner, h.pic = "", nil
|
||||
}
|
||||
pic, err := h.reader.Follow(claimant, seatHoldersState, claimant+"."+seatHoldersState)
|
||||
if err != nil {
|
||||
h.logf("the seat-holders view of %s is not followed: %v", claimant, err)
|
||||
return
|
||||
}
|
||||
h.owner, h.pic = claimant, pic
|
||||
h.logf("the seat-holders view of %s, the claimant of %s, is followed", claimant, seatlist.HolderSeat)
|
||||
}
|
||||
|
||||
// seatHoldersState is the seat mesh-module-assigner's view of who holds each seat (§616, §699).
|
||||
const seatHoldersState = "seat-holders"
|
||||
|
||||
// Holders are the modules the followed view names for seatName, and whether its first view is read: before it, and with
|
||||
// no claimant followed, nothing is judged (A-466y).
|
||||
func (h *holderView) Holders(seatName string) ([]string, bool) {
|
||||
h.mu.Lock()
|
||||
pic := h.pic
|
||||
h.mu.Unlock()
|
||||
if pic == nil {
|
||||
return nil, false
|
||||
}
|
||||
select {
|
||||
case <-pic.Follower().Ready():
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
return seatlist.HoldersIn(func() (map[string]json.RawMessage, error) { return pic.Follower().Values(), nil })(seatName), true
|
||||
}
|
||||
|
||||
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
|
||||
@@ -88,7 +154,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
|
||||
return nil, err
|
||||
}
|
||||
reader.Start(ctx, js, cred, status)
|
||||
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings}, nil
|
||||
return &runnerViews{Registrations: regs, Catalog: cv, Findings: findings, Holders: &holderView{reader: reader, catalog: cv, logf: logf}}, nil
|
||||
}
|
||||
|
||||
// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it.
|
||||
|
||||
@@ -9,6 +9,8 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -53,7 +54,12 @@ type Ager struct {
|
||||
Get(key string) (seat.KeptResult, bool, error)
|
||||
Delete(key string) error
|
||||
}
|
||||
Now func() time.Time
|
||||
// Runs is the state test-runs, RunOutputs deletes an object of test-runs-objects and RunsKept is the setting
|
||||
// test-runs-kept: an ended run is deleted with its output object once RunsKept has passed since its end (§725, A-470a).
|
||||
Runs TestRuns
|
||||
RunOutputs func(name string) error
|
||||
RunsKept time.Duration
|
||||
Now func() time.Time
|
||||
// Logf is the runner's journal: each delete writes a line naming the key or object and its retention.
|
||||
Logf func(string, ...any)
|
||||
}
|
||||
@@ -69,7 +75,16 @@ func (a *Ager) Age() {
|
||||
if a == nil || a.Log == nil {
|
||||
return
|
||||
}
|
||||
for _, r := range []Retention{Retention7, Retention30, Retention90} {
|
||||
retentions := []Retention{Retention7, Retention30, Retention90}
|
||||
if a.Runs != nil {
|
||||
// The test runs age first, in the same pass, so the pipeline that names a run's key is still kept (A-470a).
|
||||
kept := a.RunsKept
|
||||
if kept <= 0 {
|
||||
kept = DefaultRunSettings.Kept
|
||||
}
|
||||
retentions = append([]Retention{{RetentionTestRuns, kept}}, retentions...)
|
||||
}
|
||||
for _, r := range retentions {
|
||||
if err := a.ageAt(r); err != nil {
|
||||
a.logf("aged: the data past %s are not aged: %v", r.Name, err)
|
||||
}
|
||||
@@ -103,8 +118,57 @@ func (a *Ager) ageAt(r Retention) error {
|
||||
return a.Aged.Put(r.Name, moved)
|
||||
}
|
||||
|
||||
// RetentionTestRuns is the name, in the state aged, of the test runs' retention, the setting test-runs-kept (§725).
|
||||
const RetentionTestRuns = "test-runs"
|
||||
|
||||
// ageRun deletes the test run the ended step of e read, and its output object, once its kept time has passed since the
|
||||
// step's end, in the step log's order (§725, A-470a): the run's key is read from e's pipeline, its step's Shared run.
|
||||
func (a *Ager) ageRun(r Retention, e seat.LogEntry) {
|
||||
if e.What != seat.LogEntryWhatEnded {
|
||||
return
|
||||
}
|
||||
p, _, err := a.Pipelines.Get(e.Pipeline)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
run := ""
|
||||
for _, s := range p.Steps {
|
||||
if s.Step == e.Step && s.Artifact == e.Artifact {
|
||||
run = sharedRun(s)
|
||||
}
|
||||
}
|
||||
if run == "" {
|
||||
return
|
||||
}
|
||||
for {
|
||||
t, rev, err := a.Runs.Get(run)
|
||||
// The entry is past the retention (ageAt), and a run ends no later than the reader's end it was read at: a run not
|
||||
// ended yet is aged at a later reader's end.
|
||||
if err != nil || t.Ended == nil {
|
||||
return
|
||||
}
|
||||
if t.Output != "" && a.RunOutputs != nil {
|
||||
if err := a.RunOutputs(t.Output); err != nil {
|
||||
a.logf("aged: the output %s of the test run %s is not deleted, so the run is kept: %v", t.Output, run, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Deleted at the revision read: a rerun that wrote the key again since is read again, and a run it started is kept.
|
||||
if err := a.Runs.Delete(run, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
a.logf("aged: the test run %s is not deleted: %v", run, err)
|
||||
return
|
||||
}
|
||||
a.logf("aged: deleted the test run %s and its output (%s after its end)", run, r.Age)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Ager) ageEntry(r Retention, e seat.LogEntry, now time.Time) {
|
||||
switch r.Name {
|
||||
case RetentionTestRuns:
|
||||
a.ageRun(r, e)
|
||||
case Retention7.Name:
|
||||
if e.Output != "" && a.Outputs != nil {
|
||||
if err := a.Outputs(e.Output); err != nil {
|
||||
|
||||
@@ -38,7 +38,7 @@ func approvalDue(p seat.Pipeline) int {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s.Outcome == seat.StepResultOutcomeWaiting || s.Outcome == seat.StepResultOutcomeRunning {
|
||||
if s.Outcome == seat.StepResultOutcomeWaiting || open(s) {
|
||||
return -1
|
||||
}
|
||||
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
||||
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
||||
)
|
||||
|
||||
// AskBound is how long an ask on a build seat's work queue may wait for its holder to take it (design 47 G §535, §647): five
|
||||
// minutes, the bound of a key of the state `bounds` with its own time to live (A-77, A-104), never a timer.
|
||||
const AskBound = 5 * time.Minute
|
||||
// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes,
|
||||
// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as
|
||||
// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725).
|
||||
const DropBound = 5 * time.Minute
|
||||
|
||||
// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it.
|
||||
type TakenSeat struct {
|
||||
@@ -23,15 +22,15 @@ type TakenSeat struct {
|
||||
Subject string
|
||||
}
|
||||
|
||||
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken,
|
||||
// so no bound is armed for their asks.
|
||||
// TakenSeats are the three build seats, from their contracts: the seat mesh-builds and the seat mesh-wants say no ask-taken.
|
||||
var TakenSeats = []TakenSeat{
|
||||
{Seat: nodebuildgo.Seat, Subject: nodebuildgo.SubjectAskTakenAll},
|
||||
{Seat: nodebuildnode.Seat, Subject: nodebuildnode.SubjectAskTakenAll},
|
||||
{Seat: nodebuildimage.Seat, Subject: nodebuildimage.SubjectAskTakenAll},
|
||||
}
|
||||
|
||||
// BoundSeat says whether the holder of a seat says ask-taken, so an ask put on its work queue has a bound.
|
||||
// BoundSeat says whether the holder of a seat says ask-taken: a build seat, whose asks are dropped and whose test steps share
|
||||
// a run (§724).
|
||||
func BoundSeat(name string) bool {
|
||||
for _, s := range TakenSeats {
|
||||
if s.Seat == name {
|
||||
@@ -41,63 +40,6 @@ func BoundSeat(name string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Arm puts a bound of one ask's wait for its holder: a key of the state `bounds` named for the ask, written with its own
|
||||
// time to live. It is armed before the ask is put, so an ask-taken heard at once finds it.
|
||||
type Arm func(ask string) error
|
||||
|
||||
// NotTaken ends the step asked under ask that no holder took within its bound, the bound's expiry heard (design 47 G §32,
|
||||
// §535, §647, A-77): the step ends `could not run` naming the seat, so its pipeline ends failed once the others end, never
|
||||
// a silent wait. A step that ended or was never asked of a seat takes nothing.
|
||||
func (h *Hearer) NotTaken(ask string, bound time.Duration) error {
|
||||
pkeys, err := h.Store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := h.Store.Get(key)
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || stepOfAsk(p, ask) < 0 {
|
||||
continue
|
||||
}
|
||||
return h.endNotTaken(key, ask, bound)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Hearer) endNotTaken(key, ask string, bound time.Duration) error {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the pipeline %s is not read: %w", key, err)
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
s.Outcome, s.Ended = seat.StepResultOutcomeCouldNotRun, &now
|
||||
s.Why = fmt.Sprintf("the seat %s has no holder that took ask %s within %s", s.Seat, ask, bound)
|
||||
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
|
||||
ended := endPipeline(&p, now)
|
||||
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("pipeline %s cannot be written: %w", key, err)
|
||||
}
|
||||
h.Failures.StepEnded(h.Store, p, *s)
|
||||
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
||||
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
|
||||
}
|
||||
h.appendUnlogged(key)
|
||||
h.Ager.Age()
|
||||
h.Fills.Check()
|
||||
return h.said(p, false, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// IsAskKey says whether a key of the state `bounds` is an ask's bound.
|
||||
func IsAskKey(key string) bool { return strings.HasPrefix(key, "step-") && !strings.Contains(key, ".") }
|
||||
|
||||
// DropBoundPrefix begins the key of a superseded step's drop in the state `bounds`, `drop.<build seat>.<ask>` (design 47 G
|
||||
// A-222, A-77): its five-minute expiry says the drop unanswered.
|
||||
const DropBoundPrefix = "drop."
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) {
|
||||
}
|
||||
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
|
||||
said = nil
|
||||
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil, nil); n != 1 || err != nil {
|
||||
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil {
|
||||
t.Fatalf("supersede %d (%v)", n, err)
|
||||
}
|
||||
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
|
||||
|
||||
@@ -77,7 +77,18 @@ func startOf(p seat.Pipeline, now time.Time) time.Time {
|
||||
// could not run naming the missing seat; the module's steps wait. It returns the steps and the asks to put on the seats'
|
||||
// work queues, one per running step.
|
||||
func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string) ([]seat.StepResult, []Asked) {
|
||||
return place(p, reg, entries, now, asker, true)
|
||||
}
|
||||
|
||||
// place is Place; share false places each build step its own ask, as a step asked before the test runs is (§730, A-468k).
|
||||
// With share, the build steps of one candidate, level and share digest are asked once (§724): the first artifact's step, in
|
||||
// reference order, asks the run and names it in Shared; every other step of the key reads it, keeping no ask of its own and
|
||||
// naming the run's ask and the step that asked it (A-469m, A-469q).
|
||||
func place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog.CatalogEntry, now time.Time, asker string, share bool) ([]seat.StepResult, []Asked) {
|
||||
entry := entries[p.Module]
|
||||
candidate, named := CandidateOf(p)
|
||||
share = share && named
|
||||
askOfRun := map[string]int{} // a test-runs key to the index of its ask in asks
|
||||
arts := append([]catalog.Artifact(nil), entry.Artifacts...)
|
||||
sort.Slice(arts, func(i, j int) bool { return arts[i].Ref < arts[j].Ref })
|
||||
var steps []seat.StepResult
|
||||
@@ -118,8 +129,27 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog
|
||||
}
|
||||
s.Seat, s.Outcome, s.Started = bs, seat.StepResultOutcomeRunning, &now
|
||||
s.Ask = AskID(startOf(p, now), p.Pipeline, a.Ref, step)
|
||||
body := stepAsk(p, entry, a, step, s.Ask, asker, now)
|
||||
ask := Asked{Seat: bs, ID: s.Ask, Ask: body}
|
||||
if share && BoundSeat(bs) {
|
||||
digest := ShareDigest(p.Fingerprint, step, status, body.Step.Fetch)
|
||||
run := TestRunKey(candidate, step, digest)
|
||||
reader := seat.TestRunReader{Pipeline: p.Pipeline, Artifact: a.Ref}
|
||||
if j, asked := askOfRun[run]; asked {
|
||||
// Another artifact's step of this pipeline asks the run: this one reads it.
|
||||
first := &asks[j]
|
||||
first.Readers = append(first.Readers, reader)
|
||||
s.Ask, s.Outcome, s.On = "", seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: first.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: first.Readers[0].Artifact}}}
|
||||
steps = append(steps, s)
|
||||
continue
|
||||
}
|
||||
// A test run's steps wait for a build node from its ask until the holder's ask-taken (A-469b).
|
||||
s.Shared, s.Outcome, s.On = []seat.Shared{{Run: run, Ask: s.Ask}}, seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
|
||||
ask.Run, ask.Level, ask.Digest, ask.Readers = run, step, digest, []seat.TestRunReader{reader}
|
||||
askOfRun[run] = len(asks)
|
||||
}
|
||||
steps = append(steps, s)
|
||||
ask := Asked{Seat: bs, ID: s.Ask, Ask: stepAsk(p, entry, a, step, s.Ask, asker, now)}
|
||||
if a.Package != nil {
|
||||
ask.Package = a.Package.Name
|
||||
}
|
||||
@@ -156,6 +186,10 @@ type Asked struct {
|
||||
// Package is the name of the artifact's package, from the catalog entry, for a step on a build seat; its kept results
|
||||
// are the ask's (§25, §416).
|
||||
Package string
|
||||
// Run is the test run the ask asks, its key in test-runs, with its level, its share digest and the steps of the pipeline
|
||||
// reading it, the asking step first (§724, §725); empty for an ask of no test run.
|
||||
Run, Level, Digest string
|
||||
Readers []seat.TestRunReader
|
||||
}
|
||||
|
||||
func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step, id, asker string, now time.Time) nodebuildgo.StepAsk {
|
||||
@@ -170,8 +204,9 @@ func stepAsk(p seat.Pipeline, e catalog.CatalogEntry, a catalog.Artifact, step,
|
||||
Module: p.Module, Pipeline: p.Pipeline, Revision: p.Source, Step: step}}
|
||||
}
|
||||
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put.
|
||||
func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) error {
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
|
||||
// holder with no bound of the runner's own (§535, §647 as A-466y amends them).
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery bool) error {
|
||||
var first error
|
||||
for _, a := range asks {
|
||||
accept, ok := acceptStep[a.Seat]
|
||||
@@ -179,15 +214,6 @@ func askAll(asker StepAsker, arm Arm, key string, asks []Asked, recovery bool) e
|
||||
accept = to
|
||||
}
|
||||
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
|
||||
if ok {
|
||||
// A build seat's holder says ask-taken: the ask's bound is armed before it is put (§535, §647).
|
||||
if arm != nil && BoundSeat(a.Seat) {
|
||||
if aerr := arm(a.ID); aerr != nil {
|
||||
err = fmt.Errorf("its bound is not armed: %w", aerr)
|
||||
ok = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
err = asker.AskStep(accept, a.ID, a.Ask)
|
||||
}
|
||||
|
||||
@@ -328,3 +328,207 @@ func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) {
|
||||
t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks))
|
||||
}
|
||||
}
|
||||
|
||||
// testRunsMem is the state test-runs in memory, a write by compare-and-set on the revision read.
|
||||
type testRunsMem map[string]struct {
|
||||
r Run
|
||||
rev uint64
|
||||
}
|
||||
|
||||
func (m testRunsMem) Create(key string, r Run) error {
|
||||
if _, ok := m[key]; ok {
|
||||
return ErrExists
|
||||
}
|
||||
m[key] = struct {
|
||||
r Run
|
||||
rev uint64
|
||||
}{r, 1}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m testRunsMem) Get(key string) (Run, uint64, error) {
|
||||
e, ok := m[key]
|
||||
if !ok {
|
||||
return Run{}, 0, ErrNoRun
|
||||
}
|
||||
e.r.Readers = append([]RunReader(nil), e.r.Readers...)
|
||||
return e.r, e.rev, nil
|
||||
}
|
||||
|
||||
func (m testRunsMem) Delete(key string, rev uint64) error {
|
||||
if e, ok := m[key]; ok && e.rev != rev {
|
||||
return ErrChanged
|
||||
}
|
||||
delete(m, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m testRunsMem) Update(key string, r Run, rev uint64) (uint64, error) {
|
||||
if m[key].rev != rev {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
m[key] = struct {
|
||||
r Run
|
||||
rev uint64
|
||||
}{r, rev + 1}
|
||||
return rev + 1, nil
|
||||
}
|
||||
|
||||
// One test run per candidate, level and tested packages (design 47 G §724, A-469c): a module whose program and tools bundle
|
||||
// test one directory asks one lint, one unit and one integration step, the other artifact's steps reading them; a second
|
||||
// pipeline of the same candidate reads the runs; a run ended could not run is asked again; a second revision with the same
|
||||
// digests asks its own runs.
|
||||
func TestATestRunIsAskedOncePerCandidateLevelAndDigest(t *testing.T) {
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
|
||||
entries := map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
}
|
||||
store, runs, asks := memStore{}, testRunsMem{}, &stepAsks{}
|
||||
s := &Starter{Pipelines: memCreated{store}, Store: store, TestRuns: runs, Say: func(string, any) error { return nil }, Steps: asks,
|
||||
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
|
||||
start := func(source, revision int64) seat.Pipeline {
|
||||
t.Helper()
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
|
||||
t.Fatalf("start %d = %v, %v", source, ok, err)
|
||||
}
|
||||
p, _, _ := store.Get(Key("main", source, "app"))
|
||||
return p
|
||||
}
|
||||
testAsks := func(from int) []string {
|
||||
var out []string
|
||||
for _, a := range asks.asks[from:] {
|
||||
if b, ok := a.(nodebuildgo.StepAsk); ok {
|
||||
out = append(out, b.Step.Artifact+" "+b.Step.Step)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
p := start(4, 9)
|
||||
if got := strings.Join(testAsks(0), "|"); got != "app:program lint|app:program unit|app:program integration" {
|
||||
t.Fatalf("the test steps asked = %q, want the program's three alone", got)
|
||||
}
|
||||
for _, level := range buildSteps {
|
||||
asker, reader := stepNamed(p, "app:program", level), stepNamed(p, "app:tools", level)
|
||||
if asker.Ask == "" || len(asker.Shared) != 1 || asker.Shared[0].Ask != asker.Ask || asker.Shared[0].AskedBy != nil {
|
||||
t.Errorf("the asking %s step = %+v, want its ask named in shared with no asked-by", level, asker)
|
||||
}
|
||||
if reader.Ask != "" || reader.Outcome != seat.StepResultOutcomeWaitingForABuildNode || reader.On != seat.WaitOnBuildNode || len(reader.Shared) != 1 || reader.Shared[0] != (seat.Shared{
|
||||
Run: asker.Shared[0].Run, Ask: asker.Ask, AskedBy: reader.Shared[0].AskedBy}) || reader.Shared[0].AskedBy == nil ||
|
||||
*reader.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
|
||||
t.Errorf("the reading %s step = %+v, want no ask and the program's run named", level, reader)
|
||||
}
|
||||
r, _, err := runs.Get(asker.Shared[0].Run)
|
||||
if err != nil || r.Ask != asker.Ask || r.Level != level || len(r.Readers) != 2 || !strings.HasPrefix(asker.Shared[0].Run, "main.9."+level+".") {
|
||||
t.Errorf("the run of %s = %+v, %v: want the program's ask and both artifacts reading it", level, r, err)
|
||||
}
|
||||
}
|
||||
// A second pipeline of the same candidate (another source revision of the same catalog revision) reads the runs.
|
||||
n := len(asks.asks)
|
||||
q := start(5, 9)
|
||||
if got := testAsks(n); len(got) != 0 {
|
||||
t.Errorf("the second pipeline of the candidate asked %q, want it to read the runs", got)
|
||||
}
|
||||
unit := stepNamed(q, "app:program", "unit")
|
||||
if unit.Ask != "" || unit.Shared[0].Ask != stepNamed(p, "app:program", "unit").Ask || *unit.Shared[0].AskedBy != (seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: "app:program"}) {
|
||||
t.Errorf("the second pipeline's unit step = %+v, want it reading the first's run", unit)
|
||||
}
|
||||
if r, _, _ := runs.Get(unit.Shared[0].Run); len(r.Readers) != 4 {
|
||||
t.Errorf("the unit run's readers = %+v, want both pipelines' two artifacts", r.Readers)
|
||||
}
|
||||
// A run ended could not run is never read: the next step of its key asks it again.
|
||||
key := stepNamed(p, "app:program", "lint").Shared[0].Run
|
||||
r, rev, _ := runs.Get(key)
|
||||
ended := time.Unix(9, 0)
|
||||
r.Ended, r.Outcome = &ended, seat.StepResultOutcomeCouldNotRun
|
||||
_, _ = runs.Update(key, r, rev)
|
||||
n = len(asks.asks)
|
||||
start(6, 9)
|
||||
if got := strings.Join(testAsks(n), "|"); got != "app:program lint" {
|
||||
t.Errorf("after a lint run that could not run, asked %q, want the lint step alone", got)
|
||||
}
|
||||
// A second revision with the same digests asks its own runs, so a failed test runs again.
|
||||
n = len(asks.asks)
|
||||
start(7, 10)
|
||||
if got := len(testAsks(n)); got != 3 {
|
||||
t.Errorf("a second revision asked %d test steps, want its own three", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A seat with no holder is read only from the seat-holders view (design 47 G §535, §647 as A-466y and A-466z amend them):
|
||||
// once that view's first view is read, a build step whose seat it names no holder for ends could not run naming the seat,
|
||||
// and the pipeline ends failed at once naming it, every other step `not asked`, each said, none asked (A-470g, A-470h);
|
||||
// before it is read, or with a holder named, the step is asked and waits with no bound of the runner's own; the claimant of
|
||||
// the view is the one module the catalog halves name.
|
||||
func TestASeatWithNoHolderIsReadFromTheSeatHoldersView(t *testing.T) {
|
||||
reg, entries := placeFixture()
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
holders []string
|
||||
read bool
|
||||
asked bool
|
||||
}{{"the view not read yet", nil, false, true}, {"a holder named", []string{"mesh-build-go"}, true, true}, {"no holder named", nil, true, false}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
store, asks := memStore{}, &stepAsks{}
|
||||
var said []int
|
||||
s := &Starter{Pipelines: memCreated{store}, Say: func(string, any) error { return nil }, Steps: asks, Name: "the-runner",
|
||||
Registration: func(string) (catalog.Registration, bool) { return reg, true },
|
||||
Holders: func(string) ([]string, bool) { return c.holders, c.read },
|
||||
Ended: func(_ seat.Pipeline, steps []int) error { said = steps; return nil }}
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: 4, Module: "app", Commit: "c1", Fp: "src2:aa", Entries: entries}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
p, _, _ := store.Get("main.4.app")
|
||||
lint := stepNamed(p, "app:program", "lint")
|
||||
asked := false
|
||||
for _, id := range asks.ids {
|
||||
asked = asked || id == lint.Ask
|
||||
}
|
||||
if c.asked && (lint.Outcome != seat.StepResultOutcomeRunning || !asked) {
|
||||
t.Errorf("the lint step = %+v, asked %v; want it running and asked", lint, asked)
|
||||
}
|
||||
if c.asked {
|
||||
if p.Outcome != seat.PipelineOutcomeRunning || said != nil {
|
||||
t.Errorf("the pipeline = %s (%s), ended steps said %v; want it running", p.Outcome, p.Why, said)
|
||||
}
|
||||
return
|
||||
}
|
||||
if lint.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(lint.Why, "node-build-go has no holder") || len(asks.ids) != 0 {
|
||||
t.Errorf("the lint step = %+v, %d asked; want could not run naming the seat, nothing asked", lint, len(asks.ids))
|
||||
}
|
||||
if want := "the seat node-build-go has no holder: the seat-holders view names none"; p.Outcome != seat.PipelineOutcomeFailed || p.Why != want || p.Ended == nil || p.EndCount != 1 {
|
||||
t.Errorf("the pipeline = %s (%q), ended %v, count %d; want failed at once, %q", p.Outcome, p.Why, p.Ended, p.EndCount, want)
|
||||
}
|
||||
ended := 0
|
||||
for _, st := range p.Steps {
|
||||
switch {
|
||||
case st.Seat == "node-build-go", st.Outcome == seat.StepResultOutcomeNotAsked:
|
||||
ended++
|
||||
case st.Outcome == seat.StepResultOutcomeBypassed, st.Outcome == seat.StepResultOutcomeCouldNotRun && st.Seat == "":
|
||||
// ended by Place itself: a bypass, or a language no build seat names
|
||||
default:
|
||||
t.Errorf("the step %s %s = %s; want not asked", st.Artifact, st.Step, st.Outcome)
|
||||
}
|
||||
}
|
||||
if ended < 3 || len(said) != ended {
|
||||
t.Errorf("%d steps ended at placing, %d said; want each said", ended, len(said))
|
||||
}
|
||||
})
|
||||
}
|
||||
claims := func(seats ...string) map[string]any {
|
||||
var cs []any
|
||||
for _, n := range seats {
|
||||
cs = append(cs, map[string]any{"name": n, "scope": "mesh"})
|
||||
}
|
||||
return map[string]any{"module": "x", "version": "0.1", "claims": cs}
|
||||
}
|
||||
half := Half{Entries: map[string]catalog.CatalogEntry{"assigner": {Module: "assigner", Manifest: claims("mesh-module-assigner", "mesh-wants")},
|
||||
"app": {Module: "app", Manifest: claims()}}}
|
||||
if got, ok := ClaimantOf([]Half{half, {}}, "mesh-module-assigner"); !ok || got != "assigner" {
|
||||
t.Errorf("the claimant = %q, %v", got, ok)
|
||||
}
|
||||
other := Half{Entries: map[string]catalog.CatalogEntry{"other": {Module: "other", Manifest: claims("mesh-module-assigner")}}}
|
||||
if got, ok := ClaimantOf([]Half{half, other}, "mesh-module-assigner"); ok {
|
||||
t.Errorf("halves that disagree name %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,12 +95,12 @@ func (r Reads) Stands() (seat.StandsAnswer, error) {
|
||||
if s.Ended != nil || s.Started == nil {
|
||||
continue
|
||||
}
|
||||
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
if !open(s) && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
out = append(out, seat.Wait{Pipeline: p.Pipeline, Module: p.Module, Branch: p.Branch, Commit: p.Commit,
|
||||
Group: p.Group, Run: p.Run, Source: p.Source, Revision: p.Revision, Fingerprint: p.Fingerprint, Step: s.Step,
|
||||
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1})
|
||||
Artifact: s.Artifact, On: s.On, Since: *s.Started, V: 1, Ask: askOfWait(s), Shared: sharedOfWait(s)})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Since.Before(out[j].Since) })
|
||||
@@ -219,3 +219,19 @@ func startedOf(p seat.Pipeline) time.Time {
|
||||
}
|
||||
return *p.Started
|
||||
}
|
||||
|
||||
// askOfWait is a waiting step's own ask (design 47 G §731, §496): a step reading another's test run keeps none (A-469m).
|
||||
func askOfWait(s seat.StepResult) string { return s.Ask }
|
||||
|
||||
// sharedOfWait is the run a waiting step waits on, where the runner knows it (§731; A-469r, A-469v): a test step's run key
|
||||
// and ask, as one entry with no machine and no asker.
|
||||
func sharedOfWait(s seat.StepResult) []seat.Shared {
|
||||
if len(s.Shared) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]seat.Shared, 0, len(s.Shared))
|
||||
for _, x := range s.Shared {
|
||||
out = append(out, seat.Shared{Run: x.Run, Ask: x.Ask, Machine: x.Machine})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
@@ -150,3 +151,55 @@ func TestShowAndKeptCarryTheStoresFill(t *testing.T) {
|
||||
t.Errorf("show with the fill unread: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A step that reads a test run says its own step-started at every place it is asked, naming the run's key and ask in shared
|
||||
// and no asker (design 47 G §726, §729; A-469a, A-469q), and stands names each waiting step's own ask and, for a test step,
|
||||
// the run it waits on (§731; A-469r, A-469v).
|
||||
func TestStandsAndStepStartedNameWhatAReadingStepWaitsOn(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
asker, reader := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:tools", "unit")
|
||||
started := map[string]seat.StepResult{}
|
||||
for _, e := range f.said {
|
||||
started[e.Result.Artifact+" "+e.Result.Step] = e.Result
|
||||
}
|
||||
for _, s := range []seat.StepResult{asker, reader} {
|
||||
got, ok := started[s.Artifact+" unit"]
|
||||
if !ok || len(got.Shared) != 1 || got.Shared[0] != (seat.Shared{Run: asker.Shared[0].Run, Ask: asker.Ask}) {
|
||||
t.Errorf("the step-started of %s = %+v (said %v), want the run's key and ask, no asker", s.Artifact, got, ok)
|
||||
}
|
||||
}
|
||||
if _, ok := started[" composition"]; !ok {
|
||||
t.Error("the composition step's step-started is not said")
|
||||
}
|
||||
pipes := memKept{}
|
||||
put(t, pipes, p.Pipeline, f.pipeline(p))
|
||||
a, err := Reads{Pipelines: pipes, Kept: memKept{}}.Stands()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waits := map[string]seat.Wait{}
|
||||
for _, w := range a.Stands {
|
||||
waits[w.Artifact+" "+w.Step] = w
|
||||
}
|
||||
want := []seat.Shared{{Run: asker.Shared[0].Run, Ask: asker.Ask}}
|
||||
if w := waits["app:program unit"]; w.Ask != asker.Ask || len(w.Shared) != 1 || w.Shared[0] != want[0] {
|
||||
t.Errorf("the asking step stands as %+v, want its ask and its run", w)
|
||||
}
|
||||
if w := waits["app:tools unit"]; w.Ask != "" || len(w.Shared) != 1 || w.Shared[0] != want[0] {
|
||||
t.Errorf("the reading step stands as %+v, want no ask of its own and the run's key and ask", w)
|
||||
}
|
||||
if w := waits[" composition"]; w.Ask == "" || len(w.Shared) != 0 {
|
||||
t.Errorf("the composition step stands as %+v, want its own ask and no run", w)
|
||||
}
|
||||
// Resume asks the steps again and says their step-started again (A-469a).
|
||||
n := len(f.said)
|
||||
if _, err := f.s.Resume(f.store, "main", Half{Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.said) == n {
|
||||
t.Error("resume said no step-started")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,14 @@ package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Status is a step's status for one artifact of a module, read from the module's registration and never from a
|
||||
@@ -212,3 +214,35 @@ func (r *Readiness) Observe(rev int64) (first, advanced bool) {
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
// ClaimantOf is the module the catalog halves name as the claimant of seatName (design 47 G §614, A-82): the one module whose
|
||||
// entry's manifest claims it, the same in every half that names one; none, more than one, or halves that disagree name none.
|
||||
func ClaimantOf(halves []Half, seatName string) (string, bool) {
|
||||
found := ""
|
||||
for _, h := range halves {
|
||||
var named []string
|
||||
for _, e := range h.Entries {
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == seatName {
|
||||
named = append(named, e.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(named) == 0:
|
||||
continue
|
||||
case len(named) > 1, found != "" && found != named[0]:
|
||||
return "", false
|
||||
}
|
||||
found = named[0]
|
||||
}
|
||||
return found, found != ""
|
||||
}
|
||||
|
||||
@@ -229,9 +229,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
|
||||
}
|
||||
p.Fingerprint, p.Repository, p.Revision = e.Fp, e.Repository, revision
|
||||
var asks []Asked
|
||||
var ended []int
|
||||
if s.Registration != nil {
|
||||
if reg, ok := s.Registration(p.Module); ok {
|
||||
p.Steps, asks = Place(p, reg, half.Entries, now, s.Name)
|
||||
asks, ended = s.holderless(&p, asks, now)
|
||||
} else {
|
||||
s.logf("%s has no registration yet: its pipeline is placed with no step", key)
|
||||
}
|
||||
@@ -246,9 +248,11 @@ func (s *Starter) placeRunPipeline(key, id string, revision int64, half Half) (b
|
||||
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, p.Commit), ev); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", key, err)
|
||||
}
|
||||
if len(ended) > 0 {
|
||||
return true, s.endedAtPlacing(p, ended)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
|
||||
if err := s.askSteps(key, p.Commit, asks); err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,7 +79,13 @@ func TestRunOnRequest(t *testing.T) {
|
||||
if _, _, err := store.Get(RunKey("fix/1", a.Run, "same")); err == nil {
|
||||
t.Error("an entry the members do not change got a pipeline")
|
||||
}
|
||||
if len(said) != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 {
|
||||
// Each pipeline placed says pipeline-started, then step-started for each step it asks (A-469a).
|
||||
started, steps := 0, 0
|
||||
for _, x := range said {
|
||||
started += strings.Count(x, seat.EventPipelineStarted+".")
|
||||
steps += strings.Count(x, seat.EventStepStarted+".")
|
||||
}
|
||||
if started != 2 || !strings.HasPrefix(said[0], seat.EventPipelineStarted+".") || len(asks.ids) == 0 || steps < len(asks.ids) {
|
||||
t.Errorf("placing said %v and asked %d steps", said, len(asks.ids))
|
||||
}
|
||||
if ids, _ := s.UnplacedRuns(); len(ids) != 0 {
|
||||
|
||||
@@ -0,0 +1,488 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// RunSettings are the runner's settings of its test runs (design 47 G §725; A-469d, A-469w, A-469x): the cap of test-runs
|
||||
// the process judges a write by, how long an ended run is kept, the bound of a taken run's owed progress and the bound of
|
||||
// an ask never taken.
|
||||
type RunSettings struct {
|
||||
CapBytes int64
|
||||
Kept time.Duration
|
||||
StepBound time.Duration
|
||||
Untaken time.Duration
|
||||
}
|
||||
|
||||
// The settings' defaults (§725; A-466n, A-466s): 16 MiB, 7 days, 30 minutes, 24 hours.
|
||||
var DefaultRunSettings = RunSettings{CapBytes: 16 << 20, Kept: 7 * 24 * time.Hour, StepBound: 30 * time.Minute, Untaken: 24 * time.Hour}
|
||||
|
||||
// The two bound keys of a test run in test-runs, each with its own time to live (§725; A-468b, A-470): `bound.<run key>`,
|
||||
// written at the build seat's ask-taken of the run's ask and again at each step-progress of it, and `untaken.<run key>`,
|
||||
// written at the run's start.
|
||||
const (
|
||||
BoundPrefix = "bound."
|
||||
UntakenPrefix = "untaken."
|
||||
)
|
||||
|
||||
// RunObjects is the state test-runs' own object store, test-runs-objects (A-469e): a run's output, kept while its key lives.
|
||||
type RunObjects interface {
|
||||
Put(name string, data []byte) error
|
||||
// Get reads an object; ErrNoRun when it is gone.
|
||||
Get(name string) ([]byte, error)
|
||||
Delete(name string) error
|
||||
}
|
||||
|
||||
// RunClock is the bound keys of the test runs, keys of test-runs with their own time to live, each holding the ask of the
|
||||
// run it bounds (A-470n).
|
||||
type RunClock interface {
|
||||
// Bound writes key again holding ask, with ttl as its time to live, by compare-and-set on the key's latest revision
|
||||
// (retried when another write came between); it answers the revision written.
|
||||
Bound(key, ask string, ttl time.Duration) (uint64, error)
|
||||
// Bounded reads key: the ask it holds and its revision; ok false when it is not written.
|
||||
Bounded(key string) (ask string, revision uint64, ok bool, err error)
|
||||
// Unbound deletes key; a key gone is no fault.
|
||||
Unbound(key string) error
|
||||
// UnboundAt deletes key at revision; a later write of it (another run's bound) is kept, and a key gone is no fault.
|
||||
UnboundAt(key string, revision uint64) error
|
||||
// Has says whether key is written.
|
||||
Has(key string) (bool, error)
|
||||
// Keys are test-runs' keys.
|
||||
Keys() ([]string, error)
|
||||
}
|
||||
|
||||
// ParseRunBound reads a key of test-runs as a run's bound: the run's key and whether it is the untaken bound.
|
||||
func ParseRunBound(key string) (run string, untaken, ok bool) {
|
||||
if run, ok = strings.CutPrefix(key, BoundPrefix); ok {
|
||||
return run, false, run != ""
|
||||
}
|
||||
if run, ok = strings.CutPrefix(key, UntakenPrefix); ok {
|
||||
return run, true, run != ""
|
||||
}
|
||||
return "", false, false
|
||||
}
|
||||
|
||||
// IsRunBound says whether a key of test-runs is a bound key rather than a run.
|
||||
func IsRunBound(key string) bool {
|
||||
_, _, ok := ParseRunBound(key)
|
||||
return ok
|
||||
}
|
||||
|
||||
// runOfAsk is the test run asked under ask: the run its asking step names in Shared, read from the state pipelines.
|
||||
func (h *Hearer) runOfAsk(ask string) (string, bool) {
|
||||
if h.TestRuns == nil || ask == "" {
|
||||
return "", false
|
||||
}
|
||||
pkeys, err := h.Store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := h.Store.Get(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if i := stepOfAsk(p, ask); i >= 0 {
|
||||
run := sharedRun(p.Steps[i])
|
||||
return run, run != ""
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// RunTaken takes the ask-taken of a test run's ask (§725, A-470): the node that took it is kept on the run, its untaken
|
||||
// bound deleted and its clock begun, `bound.<run key>` written with test-step-bound as its time to live. An ask of no test
|
||||
// run takes nothing.
|
||||
func (h *Hearer) RunTaken(ask, node string) error {
|
||||
run, ok := h.runOfAsk(ask)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
r, rev, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
if r.Ask != ask || r.Ended != nil {
|
||||
h.logf("the ask-taken of ask %s is late: the test run %s names ask %s (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
|
||||
return nil
|
||||
}
|
||||
if r.Node != node {
|
||||
r.Node = node
|
||||
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("the test run %s is not written: %w", run, err)
|
||||
}
|
||||
}
|
||||
h.runStarted(run, r)
|
||||
break
|
||||
}
|
||||
if err := armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound); err != nil {
|
||||
return fmt.Errorf("the clock of the test run %s is not begun: %w", run, err)
|
||||
}
|
||||
// The untaken bound is deleted only while it holds this ask: a rerun's own untaken bound is kept.
|
||||
held, rev, ok, err := h.Clock.Bounded(UntakenPrefix + run)
|
||||
if err != nil || !ok || held != ask {
|
||||
return err
|
||||
}
|
||||
return h.Clock.UnboundAt(UntakenPrefix+run, rev)
|
||||
}
|
||||
|
||||
// armBound writes the bound key of the run under ask holding that ask, judged from a read of the run, then reads the run
|
||||
// again (A-470n): when it no longer names ask, or has ended, the bound just written is deleted at its revision, so no
|
||||
// bound written from the earlier read ends the run after it, and a bound another run wrote since is kept.
|
||||
func armBound(clock RunClock, runs TestRuns, key, run, ask string, ttl time.Duration) error {
|
||||
rev, err := clock.Bound(key, ask, ttl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r, _, err := runs.Get(run)
|
||||
if err != nil && !errors.Is(err, ErrNoRun) {
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, ErrNoRun) || r.Ended != nil || r.Ask != ask {
|
||||
return clock.UnboundAt(key, rev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunProgress takes a build seat's step-progress of a test run's ask (§725, A-466p): its clock begins again, so a run that
|
||||
// keeps saying its progress is never ended by it. An ask of no running, taken test run takes nothing.
|
||||
func (h *Hearer) RunProgress(ask string) error {
|
||||
run, ok := h.runOfAsk(ask)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) || (err == nil && (r.Ask != ask || r.Ended != nil)) {
|
||||
h.logf("the step-progress of ask %s is late: the test run %s names ask %q (ended %v), so it moves nothing (A-470n)", ask, run, r.Ask, r.Ended != nil)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
return armBound(h.Clock, h.TestRuns, BoundPrefix+run, run, ask, h.runSettings().StepBound)
|
||||
}
|
||||
|
||||
// RunExpired takes the expiry of a test run's bound key (§725, §720 *When the clock ends*, A-466s): the run ends `could not
|
||||
// run`, naming its ask and the bound, its ask dropped and every reader ended with it. An untaken bound of a run whose ask
|
||||
// was taken, and a bound of a run ended already, end nothing.
|
||||
func (h *Hearer) RunExpired(key string) error {
|
||||
run, untaken, ok := ParseRunBound(key)
|
||||
if !ok || h.TestRuns == nil {
|
||||
return nil
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
// An untaken bound of a taken run, and a clock of a run not taken (a clock left from an ask before a rerun's), end nothing.
|
||||
if r.Ended != nil || (untaken && r.Node != "") || (!untaken && r.Node == "") {
|
||||
return nil
|
||||
}
|
||||
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again,
|
||||
// so an expiry heard after the bound was armed again for the run's own ask ends nothing.
|
||||
if _, _, armed, err := h.Clock.Bounded(key); err != nil {
|
||||
return fmt.Errorf("the bound %s is not read again: %w", key, err)
|
||||
} else if armed {
|
||||
h.logf("the expiry of %s ends nothing: the bound is armed again (A-470o)", key)
|
||||
return nil
|
||||
}
|
||||
s := h.runSettings()
|
||||
why := fmt.Sprintf("the ask %s of %s was taken by %s and said no progress within %s (test-step-bound)", r.Ask, r.Seat, r.Node, s.StepBound)
|
||||
if untaken {
|
||||
why = fmt.Sprintf("the ask %s of %s was not taken within %s of the run's start (run-untaken-bound)", r.Ask, r.Seat, s.Untaken)
|
||||
}
|
||||
if h.DropRun != nil {
|
||||
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the test run %s is not answered: %v", r.Ask, run, err)
|
||||
}
|
||||
}
|
||||
return h.EndRun(run, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil)
|
||||
}
|
||||
|
||||
// TakeUp takes up again, at the runner's start, each test run with no end (§725): a taken run whose clock is gone, and an
|
||||
// untaken run whose untaken bound is gone, passed its bound while the runner was away and ends as its expiry would. It
|
||||
// says how many it ended.
|
||||
func (h *Hearer) TakeUp() (int, error) {
|
||||
if h.TestRuns == nil || h.Clock == nil {
|
||||
return 0, nil
|
||||
}
|
||||
all, err := h.Clock.Keys()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the test runs are not listed: %w", err)
|
||||
}
|
||||
ended := 0
|
||||
var first error
|
||||
for _, run := range all {
|
||||
if IsRunBound(run) {
|
||||
continue
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(run)
|
||||
if err != nil || r.Ended != nil {
|
||||
continue
|
||||
}
|
||||
bound := UntakenPrefix + run
|
||||
if r.Node != "" {
|
||||
bound = BoundPrefix + run
|
||||
}
|
||||
has, err := h.Clock.Has(bound)
|
||||
if err != nil || has {
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := h.RunExpired(bound); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
ended++
|
||||
}
|
||||
return ended, first
|
||||
}
|
||||
|
||||
func (h *Hearer) runSettings() RunSettings {
|
||||
s := h.Runs
|
||||
d := DefaultRunSettings
|
||||
if s.CapBytes <= 0 {
|
||||
s.CapBytes = d.CapBytes
|
||||
}
|
||||
if s.Kept <= 0 {
|
||||
s.Kept = d.Kept
|
||||
}
|
||||
if s.StepBound <= 0 {
|
||||
s.StepBound = d.StepBound
|
||||
}
|
||||
if s.Untaken <= 0 {
|
||||
s.Untaken = d.Untaken
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// EndRun ends the test run under run with end's outcome, words, counts, node and output, written once by compare-and-set,
|
||||
// and ends each of its readers with it but skip, the step whose own end ended it (§725, §726). A run ended already is not
|
||||
// ended again, and end's ask, the one its caller judged, ends only a run under that ask: a run a rerun asked since is not
|
||||
// ended by a judgement of the one before.
|
||||
func (h *Hearer) EndRun(run string, end seat.TestRun, skip *seat.TestRunReader) error {
|
||||
var r Run
|
||||
for {
|
||||
got, rev, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
if got.Ended != nil || (end.Ask != "" && got.Ask != end.Ask) {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
got.Ended, got.Outcome, got.Said, got.Tests = &now, end.Outcome, end.Said, end.Tests
|
||||
if end.Node != "" {
|
||||
got.Node = end.Node
|
||||
}
|
||||
if end.Output != "" {
|
||||
got.Output = end.Output
|
||||
}
|
||||
if _, err := h.TestRuns.Update(run, got, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("the test run %s is not written: %w", run, err)
|
||||
}
|
||||
r = got
|
||||
break
|
||||
}
|
||||
if h.Clock != nil {
|
||||
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
|
||||
if err := h.Clock.Unbound(k); err != nil {
|
||||
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
var first error
|
||||
for _, reader := range r.Readers {
|
||||
if skip != nil && reader.Of() == *skip {
|
||||
continue
|
||||
}
|
||||
if err := h.endReader(reader, run, r); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// endReader ends reader's step of run with the run's result (§726): its outcome, words, counts and node, the step naming
|
||||
// the run in Shared as it did. A step ended already takes nothing, but one the rerun that asked the run marked (A-470d,
|
||||
// A-470e): it read another ask of the key, and it moves with this one, said again marked a rerun by that caller, naming
|
||||
// the new ask, its pipeline reopened (§727).
|
||||
func (h *Hearer) endReader(reader RunReader, run string, r Run) error {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(reader.Pipeline)
|
||||
if err != nil {
|
||||
return nil // a pipeline gone has no reader left to end
|
||||
}
|
||||
i, moved := -1, false
|
||||
for j, s := range p.Steps {
|
||||
if s.Artifact != reader.Artifact || sharedRun(s) != run {
|
||||
continue
|
||||
}
|
||||
if open(s) {
|
||||
i, moved = j, false
|
||||
} else if reader.Rerun && s.Shared[0].Ask != r.Ask && s.Accepted == nil {
|
||||
i, moved = j, true
|
||||
}
|
||||
}
|
||||
if i < 0 {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
if moved {
|
||||
s.Rerun, s.By = true, reader.By
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: s.Shared[0].AskedBy}}
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
|
||||
}
|
||||
}
|
||||
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = r.Outcome, r.Said, r.Node, &now, nil
|
||||
if r.Tests != nil {
|
||||
t := *r.Tests
|
||||
s.Tests = &t
|
||||
}
|
||||
p.Unlogged = append(p.Unlogged, endEntry(reader.Pipeline, *s, now))
|
||||
asked, err := askApproval(reader.Pipeline, &p, now, h.Asker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ended := endPipeline(&p, now)
|
||||
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("pipeline %s cannot be written: %w", reader.Pipeline, err)
|
||||
}
|
||||
h.Failures.StepEnded(h.Store, p, *s)
|
||||
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
||||
return fmt.Errorf("pipeline %s's %s is written but not said: %w", reader.Pipeline, seat.EventStepEnded, err)
|
||||
}
|
||||
h.appendUnlogged(reader.Pipeline)
|
||||
return h.said(p, asked, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// NoReaderLeft is what a test run whose last reader left ends with (A-469z): its ask is dropped, and a later step of the
|
||||
// candidate starts a new run.
|
||||
const NoReaderLeft = "no reader is left"
|
||||
|
||||
// leaveRun takes the step of pipeline key that read or asked a test run out of the run's readers (§726): a step superseded
|
||||
// or cancelled. When no reader is left the run ends `could not run`, naming that (A-469z), and its ask is dropped (§119);
|
||||
// while one is left the ask stays, so a superseded pipeline never takes away another's run. A step of no test run leaves
|
||||
// nothing.
|
||||
func (h *Hearer) leaveRun(key string, s seat.StepResult, why string) {
|
||||
run := sharedRun(s)
|
||||
if run == "" || h.TestRuns == nil {
|
||||
return
|
||||
}
|
||||
reader := seat.TestRunReader{Pipeline: key, Artifact: s.Artifact}
|
||||
for {
|
||||
r, rev, err := h.TestRuns.Get(run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
h.logf("the test run %s is not read, so the step %s of %s does not leave it: %v", run, s.Step, key, err)
|
||||
return
|
||||
}
|
||||
left := r.Readers[:0:0]
|
||||
for _, o := range r.Readers {
|
||||
if o.Of() != reader {
|
||||
left = append(left, o)
|
||||
}
|
||||
}
|
||||
if len(left) == len(r.Readers) {
|
||||
return
|
||||
}
|
||||
r.Readers = left
|
||||
// The last to leave ends the run in the same write, so a step joining it after that read finds it ended and asks
|
||||
// its own, never joining a run then ended for having no reader.
|
||||
last := len(left) == 0 && r.Ended == nil
|
||||
if last {
|
||||
now := h.now()
|
||||
r.Ended, r.Outcome, r.Said = &now, seat.StepResultOutcomeCouldNotRun, NoReaderLeft
|
||||
}
|
||||
if _, err := h.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
h.logf("the step %s of %s does not leave the test run %s: %v", s.Step, key, run, err)
|
||||
return
|
||||
}
|
||||
if !last {
|
||||
return
|
||||
}
|
||||
if h.Clock != nil {
|
||||
for _, k := range []string{BoundPrefix + run, UntakenPrefix + run} {
|
||||
if err := h.Clock.Unbound(k); err != nil {
|
||||
h.logf("the bound %s of the ended test run is not deleted: %v", k, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if h.DropRun != nil {
|
||||
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the test run %s, which no reader is left of, is not answered: %v", r.Ask, run, err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// open says whether a step is asked and not ended: running, or a test step waiting for a build node to take its run's ask
|
||||
// (A-469b).
|
||||
func open(s seat.StepResult) bool {
|
||||
return s.Outcome == seat.StepResultOutcomeRunning || s.Outcome == seat.StepResultOutcomeWaitingForABuildNode
|
||||
}
|
||||
|
||||
// runStarted moves each step reading run from waiting for a build node to running, the holder's ask-taken heard (A-469b),
|
||||
// and says its step-started again, so the watchman's build-capacity waits let it go (§86, §732).
|
||||
func (h *Hearer) runStarted(run string, r Run) {
|
||||
for _, reader := range r.Readers {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(reader.Pipeline)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
var moved []int
|
||||
for j, s := range p.Steps {
|
||||
if s.Artifact == reader.Artifact && sharedRun(s) == run && s.Outcome == seat.StepResultOutcomeWaitingForABuildNode {
|
||||
p.Steps[j].Outcome, p.Steps[j].On = seat.StepResultOutcomeRunning, ""
|
||||
moved = append(moved, j)
|
||||
}
|
||||
}
|
||||
if len(moved) == 0 {
|
||||
break
|
||||
}
|
||||
if _, err := h.Store.Update(reader.Pipeline, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
h.logf("the steps of %s reading the taken test run %s are not written running: %v", reader.Pipeline, run, err)
|
||||
break
|
||||
}
|
||||
for _, j := range moved {
|
||||
if err := h.say(seat.EventStepStarted, p, &p.Steps[j]); err != nil {
|
||||
h.logf("step-started of %s's %s is not said: %v", reader.Pipeline, p.Steps[j].Step, err)
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
@@ -73,8 +75,12 @@ type Starter struct {
|
||||
// Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when
|
||||
// none was heard, which asks the step with none.
|
||||
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
|
||||
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
|
||||
Arm Arm
|
||||
// Holders are the modules the seat mesh-module-assigner's view seat-holders names for a seat, and whether that view's
|
||||
// first view is read (§535, §647 as A-466y, A-466z amend them); nil, or a view not read yet, judges no seat holderless.
|
||||
Holders func(seat string) ([]string, bool)
|
||||
// Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end
|
||||
// (Hearer.EndedAtPlacing); nil says it in the journal alone.
|
||||
Ended func(p seat.Pipeline, steps []int) error
|
||||
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
|
||||
Fills *Fills
|
||||
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
|
||||
@@ -83,6 +89,116 @@ type Starter struct {
|
||||
Store Updater
|
||||
// Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none.
|
||||
Supersede func(newer seat.Pipeline)
|
||||
// TestRuns is the state test-runs, where a test step's run is shared by every step of its candidate, level and digest
|
||||
// (§724, §725); nil shares none, each step's ask put. Clock writes a run's untaken bound at its start, Runs holds the
|
||||
// settings, RunsFill reads test-runs' fill, judged against Runs' cap before a run is written (A-469d), and EndReaders ends
|
||||
// the steps of a run read ended or never asked.
|
||||
TestRuns TestRuns
|
||||
Clock RunClock
|
||||
Runs RunSettings
|
||||
RunsFill func() (seat.Fill, error)
|
||||
EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error
|
||||
}
|
||||
|
||||
// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view
|
||||
// seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only
|
||||
// once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat,
|
||||
// every other step not ended ends `not asked`, each with its step log entry, and the pipeline ends `failed` naming the seat;
|
||||
// none of its steps is asked. It answers the asks still to put and the steps it ended, none when no seat is holderless.
|
||||
func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]Asked, []int) {
|
||||
if s.Holders == nil {
|
||||
return asks, nil
|
||||
}
|
||||
var none []string
|
||||
for _, st := range p.Steps {
|
||||
if !open(st) || !BoundSeat(st.Seat) || slices.Contains(none, st.Seat) {
|
||||
continue
|
||||
}
|
||||
if holders, read := s.Holders(st.Seat); read && len(holders) == 0 {
|
||||
none = append(none, st.Seat)
|
||||
}
|
||||
}
|
||||
if len(none) == 0 {
|
||||
return asks, nil
|
||||
}
|
||||
sort.Strings(none)
|
||||
whys := make([]string, 0, len(none))
|
||||
for _, n := range none {
|
||||
whys = append(whys, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", n))
|
||||
}
|
||||
var ended []int
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
st.Ended, st.Ask, st.Shared = &now, "", nil
|
||||
if slices.Contains(none, st.Seat) {
|
||||
st.Outcome, st.Why = seat.StepResultOutcomeCouldNotRun, fmt.Sprintf("the seat %s has no holder: the seat-holders view names none", st.Seat)
|
||||
} else {
|
||||
st.Outcome, st.Why = seat.StepResultOutcomeNotAsked, "the pipeline failed at once: "+strings.Join(whys, "; ")
|
||||
}
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, strings.Join(whys, "; "), &now
|
||||
p.EndCount++
|
||||
return nil, ended
|
||||
}
|
||||
|
||||
// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the
|
||||
// pipeline's end, through Ended; with none given it is said in the journal alone.
|
||||
func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
|
||||
s.logf("%s ended failed at once: %s (A-470g)", p.Pipeline, p.Why)
|
||||
if s.Ended == nil {
|
||||
return nil
|
||||
}
|
||||
return s.Ended(p, steps)
|
||||
}
|
||||
|
||||
// askSteps puts a placed pipeline's asks (§22, §24): a test run another step asked is read, not asked again (share), then
|
||||
// the kept results and toolchain are carried and each ask is put.
|
||||
func (s *Starter) askSteps(key, commit string, asks []Asked) error {
|
||||
placed := asks
|
||||
asks = s.share(key, asks)
|
||||
var err error
|
||||
if len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
err = askAll(s.Steps, key, asks, s.recovered(key, commit))
|
||||
}
|
||||
s.sayStarted(key, placed)
|
||||
return err
|
||||
}
|
||||
|
||||
// sayStarted says step-started for each step of pipeline key that asks were placed for (design 47 G §726, §729; A-469a):
|
||||
// the step asked, and each step reading the test run it asks, naming the run's key and ask in shared, so the watchman times
|
||||
// it from that ask (§496, §732). A composition step's, said before any run is chosen, names none (§729).
|
||||
func (s *Starter) sayStarted(key string, asks []Asked) {
|
||||
if s.Store == nil || s.Say == nil || len(asks) == 0 {
|
||||
return
|
||||
}
|
||||
ids, runs := map[string]bool{}, map[string]bool{}
|
||||
for _, a := range asks {
|
||||
ids[a.ID] = true
|
||||
if a.Run != "" {
|
||||
runs[a.Run] = true
|
||||
}
|
||||
}
|
||||
p, _, err := s.Store.Get(key)
|
||||
if err != nil {
|
||||
s.logf("the pipeline %s is not read, so its steps' step-started is not said: %v", key, err)
|
||||
return
|
||||
}
|
||||
at := s.now()
|
||||
for _, st := range p.Steps {
|
||||
if !open(st) || !(ids[st.Ask] || (st.Ask == "" && runs[sharedRun(st)])) {
|
||||
continue
|
||||
}
|
||||
subject, ev := stepEvent(seat.EventStepStarted, p, st, at)
|
||||
if err := s.Say(subject, ev); err != nil {
|
||||
s.logf("step-started of %s's %s %s is not said: %v", key, st.Artifact, st.Step, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
|
||||
@@ -115,11 +231,13 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
}
|
||||
p := newPipeline(b, key, now)
|
||||
var asks []Asked
|
||||
var ended []int
|
||||
if b.unheld {
|
||||
s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch)
|
||||
} else if s.Registration != nil {
|
||||
if reg, ok := s.Registration(b.Module); ok {
|
||||
p.Steps, asks = Place(p, reg, b.Entries, now, s.Name)
|
||||
asks, ended = s.holderless(&p, asks, now)
|
||||
} else {
|
||||
s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module)
|
||||
}
|
||||
@@ -143,9 +261,11 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
if s.Supersede != nil {
|
||||
s.Supersede(p)
|
||||
}
|
||||
if len(ended) > 0 {
|
||||
return true, s.endedAtPlacing(p, ended)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil {
|
||||
if err := s.askSteps(key, b.Commit, asks); err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
@@ -187,24 +307,32 @@ func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
running := map[string]bool{}
|
||||
// A step that names its test run is asked as the run's asker (share); one that names none was asked before the test
|
||||
// runs and keeps its own ask, ending by the path it was asked under (§730, A-468k).
|
||||
running, own := map[string]bool{}, map[string]bool{}
|
||||
for _, st := range p.Steps {
|
||||
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
|
||||
running[st.Ask] = true
|
||||
if open(st) && st.Ask != "" {
|
||||
running[st.Ask], own[st.Ask] = true, len(st.Shared) == 0
|
||||
}
|
||||
}
|
||||
_, placed := Place(p, reg, half.Entries, now, s.Name)
|
||||
_, shared := Place(p, reg, half.Entries, now, s.Name)
|
||||
_, unshared := place(p, reg, half.Entries, now, s.Name, false)
|
||||
var due []Asked
|
||||
for _, a := range placed {
|
||||
if running[a.ID] {
|
||||
for _, a := range shared {
|
||||
if running[a.ID] && !own[a.ID] {
|
||||
due = append(due, a)
|
||||
}
|
||||
}
|
||||
for _, a := range unshared {
|
||||
if running[a.ID] && own[a.ID] {
|
||||
a.Run = ""
|
||||
due = append(due, a)
|
||||
}
|
||||
}
|
||||
if len(due) == 0 {
|
||||
continue
|
||||
}
|
||||
s.carry(key, due)
|
||||
if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil {
|
||||
if err := s.askSteps(key, p.Commit, due); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
asked += len(due)
|
||||
@@ -284,6 +412,7 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
|
||||
}
|
||||
var asks []Asked
|
||||
p.Steps, asks = Place(p, r, half.Entries, now, s.Name)
|
||||
asks, ended := s.holderless(&p, asks, now)
|
||||
if _, err := store.Update(key, p, rev); err != nil {
|
||||
s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err)
|
||||
continue
|
||||
@@ -299,9 +428,14 @@ func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipe
|
||||
s.logf("pipeline-started of %s is not said: %v", key, err)
|
||||
}
|
||||
}
|
||||
if len(ended) > 0 {
|
||||
if err := s.endedAtPlacing(p, ended); err != nil {
|
||||
note(err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
|
||||
if err := s.askSteps(key, p.Commit, asks); err != nil {
|
||||
note(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ type StepSeat struct {
|
||||
Prefix string
|
||||
// Decode reads the seat's step-ended body; nil is the build seats' one form.
|
||||
Decode func([]byte) (StepEnded, error)
|
||||
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
|
||||
// a seat whose ends name none.
|
||||
Shared func([]byte) []seat.Shared
|
||||
}
|
||||
|
||||
// StepSeats are the three build seats, from their contracts.
|
||||
@@ -36,7 +39,25 @@ var StepSeats = []StepSeat{
|
||||
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
|
||||
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
|
||||
// The seat mesh-wants answers the composition step (§22, §24).
|
||||
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
|
||||
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
|
||||
}
|
||||
|
||||
// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d,
|
||||
// A-469t): the same form, under the same name.
|
||||
func wantsShared(raw []byte) []seat.Shared {
|
||||
var w wants.StepEnded
|
||||
if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]seat.Shared, 0, len(w.Shared))
|
||||
for _, x := range w.Shared {
|
||||
e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask}
|
||||
if x.AskedBy != nil {
|
||||
e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact}
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
|
||||
@@ -118,6 +139,26 @@ type Hearer struct {
|
||||
Fills *Fills
|
||||
// Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none.
|
||||
Widens func(seat.Pipeline) []seat.Widening
|
||||
// TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a
|
||||
// run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run.
|
||||
TestRuns TestRuns
|
||||
Clock RunClock
|
||||
Runs RunSettings
|
||||
DropRun Drop
|
||||
// RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v).
|
||||
RunOutputs RunObjects
|
||||
}
|
||||
|
||||
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
|
||||
// of a run that was never asked.
|
||||
func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error {
|
||||
var first error
|
||||
for _, reader := range readers {
|
||||
if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func (h *Hearer) now() time.Time {
|
||||
@@ -132,6 +173,13 @@ func (h *Hearer) now() time.Time {
|
||||
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
|
||||
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
|
||||
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
|
||||
return h.OnStepEndedShared(bs, subject, ev, nil)
|
||||
}
|
||||
|
||||
// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end
|
||||
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
|
||||
// A-469m).
|
||||
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
|
||||
node := strings.TrimPrefix(subject, bs.Prefix)
|
||||
if node == subject || node == "" || strings.Contains(node, ".") {
|
||||
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
|
||||
@@ -153,7 +201,15 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
|
||||
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
|
||||
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
|
||||
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
|
||||
case !open(p.Steps[i]):
|
||||
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
|
||||
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
|
||||
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
|
||||
return p, h.endRunOf(p, p.Steps[i], ev)
|
||||
}
|
||||
// The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725).
|
||||
h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why)
|
||||
}
|
||||
return p, nil
|
||||
case !endOutcome(ev.Outcome):
|
||||
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
|
||||
@@ -161,6 +217,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
|
||||
if len(shared) > 0 {
|
||||
s.Shared = shared
|
||||
}
|
||||
if ev.Tests != nil {
|
||||
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
||||
}
|
||||
@@ -177,6 +236,8 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
}
|
||||
unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...)
|
||||
p.Unlogged = append(p.Unlogged, unlogged)
|
||||
// The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727).
|
||||
readers := h.sameRunReaders(&p, i, ev, now)
|
||||
from := now
|
||||
if ev.Ended != nil {
|
||||
from = ev.Ended.UTC()
|
||||
@@ -191,6 +252,9 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
} else if err != nil {
|
||||
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
|
||||
}
|
||||
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
|
||||
// finding's event the bus is slow to take) delays it past its bound (A-470o).
|
||||
runErr := h.endRunOf(p, *s, ev)
|
||||
if h.Keep != nil {
|
||||
for _, k := range ev.Kept {
|
||||
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
|
||||
@@ -209,14 +273,94 @@ func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pi
|
||||
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
||||
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
|
||||
}
|
||||
for _, j := range readers {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
|
||||
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
|
||||
}
|
||||
}
|
||||
// The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606).
|
||||
h.appendUnlogged(ev.Pipeline)
|
||||
h.Ager.Age()
|
||||
h.Fills.Check()
|
||||
return p, h.said(p, asked, ended)
|
||||
if err := h.said(p, asked, ended); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return p, runErr
|
||||
}
|
||||
}
|
||||
|
||||
// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader,
|
||||
// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It
|
||||
// answers their indexes.
|
||||
func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int {
|
||||
asker := p.Steps[i]
|
||||
run := sharedRun(asker)
|
||||
if run == "" || h.TestRuns == nil || asker.Ask != ev.ID {
|
||||
return nil
|
||||
}
|
||||
marks := map[string]RunReader{}
|
||||
if r, _, err := h.TestRuns.Get(run); err == nil {
|
||||
for _, o := range r.Readers {
|
||||
if o.Pipeline == p.Pipeline {
|
||||
marks[o.Artifact] = o
|
||||
}
|
||||
}
|
||||
}
|
||||
var out []int
|
||||
for j := range p.Steps {
|
||||
s := &p.Steps[j]
|
||||
if j == i || sharedRun(*s) != run {
|
||||
continue
|
||||
}
|
||||
switch m := marks[s.Artifact]; {
|
||||
case open(*s) && s.Ask == "":
|
||||
case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil:
|
||||
s.Rerun, s.By = true, m.By
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil
|
||||
if asker.Tests != nil {
|
||||
t := *asker.Tests
|
||||
s.Tests = &t
|
||||
}
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now))
|
||||
out = append(out, j)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in
|
||||
// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it.
|
||||
// A step that asked no run ends nothing more.
|
||||
func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error {
|
||||
run := sharedRun(s)
|
||||
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
|
||||
return nil
|
||||
}
|
||||
end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
|
||||
if ev.Tests != nil {
|
||||
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
||||
}
|
||||
if h.RunOutputs != nil {
|
||||
// The run's output object holds the packages it ran with their digests and the build seat's output (§725).
|
||||
out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}}
|
||||
for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) {
|
||||
out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest})
|
||||
}
|
||||
raw, err := json.Marshal(out)
|
||||
if err == nil {
|
||||
err = h.RunOutputs.Put(run, raw)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err)
|
||||
}
|
||||
end.Output = run
|
||||
}
|
||||
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
|
||||
}
|
||||
|
||||
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
|
||||
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
|
||||
if asked {
|
||||
@@ -300,8 +444,8 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
|
||||
at := h.now()
|
||||
switch event {
|
||||
case seat.EventStepEnded, seat.EventStepStarted:
|
||||
return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
|
||||
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step})
|
||||
_, ev := stepEvent(event, p, *step, at)
|
||||
return h.Say(subject, ev)
|
||||
case seat.EventPipelineEnded:
|
||||
e := pipelineEnded(p)
|
||||
if h.Widens != nil {
|
||||
@@ -314,17 +458,46 @@ func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error
|
||||
}
|
||||
}
|
||||
|
||||
// stepEvent is a step's step-started or step-ended of p, on the runner's subject for it: its result as it stands, with
|
||||
// its shared when it reads a run (§729; A-468e, A-469q).
|
||||
func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) {
|
||||
last := p.Commit
|
||||
if p.Group != "" {
|
||||
last = p.Group
|
||||
}
|
||||
if event == seat.EventStepStarted {
|
||||
// A step-started names the run and its ask, never the step that asked it (A-469q).
|
||||
var shared []seat.Shared
|
||||
for _, x := range step.Shared {
|
||||
x.AskedBy = nil
|
||||
shared = append(shared, x)
|
||||
}
|
||||
step.Shared = shared
|
||||
}
|
||||
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
|
||||
return subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
|
||||
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: step}
|
||||
}
|
||||
|
||||
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
|
||||
const (
|
||||
budgetUnit = 500
|
||||
budgetIntegration = 50
|
||||
)
|
||||
|
||||
// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any.
|
||||
// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many
|
||||
// steps read it (§726, A-466i); nil when no step counted any.
|
||||
func testCounts(steps []seat.StepResult) *seat.TestCounts {
|
||||
var t seat.TestCounts
|
||||
counted := false
|
||||
runs := map[string]bool{}
|
||||
for _, s := range steps {
|
||||
if run := sharedRun(s); run != "" && s.Seat != wants.Seat {
|
||||
if runs[run] {
|
||||
continue
|
||||
}
|
||||
runs[run] = true
|
||||
}
|
||||
if s.Tests != nil {
|
||||
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
|
||||
}
|
||||
@@ -380,7 +553,7 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
|
||||
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
|
||||
// pipeline's end (A-104).
|
||||
continue
|
||||
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
|
||||
case s.Outcome == seat.StepResultOutcomeWaiting, open(s):
|
||||
return false
|
||||
}
|
||||
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
|
||||
@@ -401,3 +574,22 @@ func endPipeline(p *seat.Pipeline, now time.Time) bool {
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each
|
||||
// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended.
|
||||
func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error {
|
||||
var first error
|
||||
for _, i := range steps {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
|
||||
first = fmt.Errorf("pipeline %s's %s is written but not said: %w", p.Pipeline, seat.EventStepEnded, err)
|
||||
}
|
||||
}
|
||||
h.appendUnlogged(p.Pipeline)
|
||||
if len(steps) > 0 {
|
||||
h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]])
|
||||
}
|
||||
if err := h.said(p, false, true); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
@@ -302,49 +302,6 @@ func TestAnOptionalStepStillWaitingHoldsNoEnd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An ask no holder took within its bound ends its step `could not run` naming the seat, and with the rest ended the pipeline
|
||||
// failed (design 47 G §32, §535, §647, A-104); a step that ended, or ask nobody placed, ends nothing.
|
||||
func TestAnAskNoHolderTookEndsItsStepNamingTheSeat(t *testing.T) {
|
||||
h, store, said, _, _, _ := fixture()
|
||||
e := store[pkey]
|
||||
e.p.Steps[1].Outcome = seat.StepResultOutcomePassed // the unit step ended; only lint is waiting for a holder
|
||||
e.p.Steps[2].Outcome = seat.StepResultOutcomeNotAsked
|
||||
store[pkey] = e
|
||||
if err := h.NotTaken("nobody-asked", AskBound); err != nil || store[pkey].p.Steps[0].Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Fatalf("an ask no step names ended something: %v", err)
|
||||
}
|
||||
if err := h.NotTaken("a1", AskBound); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := store[pkey].p
|
||||
if p.Steps[0].Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(p.Steps[0].Why, nodebuildgo.Seat) ||
|
||||
p.Outcome != seat.PipelineOutcomeFailed || !strings.Contains(strings.Join(*said, " "), seat.EventPipelineEnded) {
|
||||
t.Errorf("step %+v, pipeline %s (%s), said %v: want could not run naming %s and a failed pipeline", p.Steps[0], p.Outcome, p.Why, *said, nodebuildgo.Seat)
|
||||
}
|
||||
// Its end taken once: the bound's expiry heard again changes nothing.
|
||||
n := len(*said)
|
||||
if err := h.NotTaken("a1", AskBound); err != nil || len(*said) != n {
|
||||
t.Errorf("a second expiry said %d more events: %v", len(*said)-n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The bound is armed before the ask is put, only for the build seats whose holder says ask-taken, and an ask whose bound is
|
||||
// not armed is not put (§535, §647).
|
||||
func TestTheBoundIsArmedBeforeAnAskIsPut(t *testing.T) {
|
||||
var order []string
|
||||
asker := askerFunc(func(accept, id string, _ any) error { order = append(order, "put "+id); return nil })
|
||||
arm := Arm(func(ask string) error { order = append(order, "arm "+ask); return nil })
|
||||
asks := []Asked{{Seat: nodebuildgo.Seat, ID: "step-1", Ask: 1}, {Seat: wants.Seat, ID: "step-2", Ask: 2}}
|
||||
if err := askAll(asker, arm, "k", asks, false); err != nil || strings.Join(order, ",") != "arm step-1,put step-1,put step-2" {
|
||||
t.Errorf("order %v, %v; want the build seat's bound armed first and mesh-wants' ask with none", order, err)
|
||||
}
|
||||
order = nil
|
||||
failing := Arm(func(string) error { return errors.New("bucket gone") })
|
||||
if err := askAll(asker, failing, "k", asks[:1], false); err == nil || len(order) != 0 {
|
||||
t.Errorf("an ask whose bound cannot be armed was put or said nothing: %v %v", order, err)
|
||||
}
|
||||
}
|
||||
|
||||
type askerFunc func(accept, id string, body any) error
|
||||
|
||||
func (f askerFunc) AskStep(accept, id string, body any) error { return f(accept, id, body) }
|
||||
@@ -359,7 +316,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
|
||||
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
|
||||
var to []string
|
||||
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
|
||||
if err := askAll(asker, nil, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
|
||||
if err := askAll(asker, "k", asks, c.recovery); err != nil || strings.Join(to, ",") != c.want {
|
||||
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ func (h *Hearer) Taken(ask, node string) error {
|
||||
break
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i < 0 || p.Steps[i].Outcome != seat.StepResultOutcomeRunning || p.Steps[i].Node == node {
|
||||
if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node {
|
||||
break
|
||||
}
|
||||
p.Steps[i].Node = node
|
||||
@@ -49,10 +49,10 @@ func (h *Hearer) Taken(ask, node string) error {
|
||||
// whose fingerprint newer's commit changed, and for a publish's pipeline each an older publish of the package started
|
||||
// (design 47 G §31, §119; ADR 0666 §9; A-202): the pipeline and each step not ended
|
||||
// `superseded`, naming the newer commit, and it says pipeline-ended. A module the newer commit left as it was finishes its
|
||||
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop; its bound deleted through
|
||||
// unbound); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
|
||||
// pipeline. The ask of each superseded step on a build seat that serves `drop` is dropped (drop); a step of a test run leaves
|
||||
// it, its ask dropped only with its last reader (§726); a step on mesh-builds or mesh-wants, whose seats serve no drop, is only ended superseded (A-182). A run on
|
||||
// request's pipeline neither supersedes nor is superseded. It says how many pipelines it ended.
|
||||
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask string) error) (int, error) {
|
||||
func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop) (int, error) {
|
||||
if newer.Run != "" || newer.Source <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -67,7 +67,7 @@ func (h *Hearer) Supersede(newer seat.Pipeline, drop Drop, unbound func(ask stri
|
||||
if key == newer.Pipeline {
|
||||
continue
|
||||
}
|
||||
done, err := h.supersedeOne(key, newer, why, drop, unbound)
|
||||
done, err := h.supersedeOne(key, newer, why, drop)
|
||||
if err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
@@ -99,7 +99,7 @@ func supersededWhy(newer seat.Pipeline) string {
|
||||
return fmt.Sprintf("the newer commit %s of %s changed %s's fingerprint", newer.Commit, newer.Branch, newer.Module)
|
||||
}
|
||||
|
||||
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop, unbound func(string) error) (bool, error) {
|
||||
func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop Drop) (bool, error) {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(key)
|
||||
if err != nil || !superseded(p, newer) {
|
||||
@@ -110,10 +110,11 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
|
||||
var changed []int
|
||||
for i := range p.Steps {
|
||||
s := &p.Steps[i]
|
||||
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
if s.Ask != "" && BoundSeat(s.Seat) {
|
||||
// A step of a test run leaves it: its ask is dropped only with its last reader (§726, leaveRun).
|
||||
if s.Ask != "" && BoundSeat(s.Seat) && sharedRun(*s) == "" {
|
||||
drops = append(drops, *s)
|
||||
}
|
||||
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeSuperseded, why, ptrTime(now)
|
||||
@@ -136,17 +137,15 @@ func (h *Hearer) supersedeOne(key string, newer seat.Pipeline, why string, drop
|
||||
if len(changed) > 0 {
|
||||
h.appendUnlogged(key)
|
||||
}
|
||||
for _, i := range changed {
|
||||
h.leaveRun(key, p.Steps[i], why)
|
||||
}
|
||||
for _, s := range drops {
|
||||
if drop != nil {
|
||||
if err := drop(s.Seat, s.Node, s.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the superseded %s is not answered: %v", s.Ask, key, err)
|
||||
}
|
||||
}
|
||||
if unbound != nil {
|
||||
if err := unbound(s.Ask); err != nil {
|
||||
h.logf("the bound of ask %s of the superseded %s is not dropped: %v", s.Ask, key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := h.said(p, false, true); err != nil && first == nil {
|
||||
first = err
|
||||
|
||||
@@ -29,7 +29,6 @@ func TestSupersede(t *testing.T) {
|
||||
}
|
||||
type dropped struct{ seat, node, ask string }
|
||||
var drops []dropped
|
||||
var unbound []string
|
||||
drop := func(s, node, ask, why string) error {
|
||||
if !strings.Contains(why, "c2") {
|
||||
t.Errorf("the drop of %s names no newer commit: %q", ask, why)
|
||||
@@ -37,7 +36,6 @@ func TestSupersede(t *testing.T) {
|
||||
drops = append(drops, dropped{s, node, ask})
|
||||
return nil
|
||||
}
|
||||
unbind := func(ask string) error { unbound = append(unbound, ask); return nil }
|
||||
newer := func(key string, source int64, fp string, run string, pub *seat.Published) seat.Pipeline {
|
||||
return seat.Pipeline{Pipeline: key, Branch: "main", Source: source, Module: "app", Commit: "c2", Fingerprint: fp, Run: run, Published: pub,
|
||||
Outcome: seat.PipelineOutcomeRunning}
|
||||
@@ -50,12 +48,12 @@ func TestSupersede(t *testing.T) {
|
||||
{"a run on request", newer("main.r1.app", 6, "src2:bb", "r1", nil)},
|
||||
{"an older source revision", newer("main.4.app", 4, "src2:bb", "", nil)},
|
||||
} {
|
||||
if n, err := h.Supersede(c.newer, drop, unbind); n != 0 || err != nil {
|
||||
if n, err := h.Supersede(c.newer, drop); n != 0 || err != nil {
|
||||
t.Errorf("%s superseded %d (%v)", c.name, n, err)
|
||||
}
|
||||
}
|
||||
*said = nil
|
||||
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop, unbind); n != 1 || err != nil {
|
||||
if n, err := h.Supersede(newer("main.6.app", 6, "src2:bb", "", nil), drop); n != 1 || err != nil {
|
||||
t.Fatalf("a changed fingerprint superseded %d (%v)", n, err)
|
||||
}
|
||||
p, _, _ := store.Get(pkey)
|
||||
@@ -68,13 +66,13 @@ func TestSupersede(t *testing.T) {
|
||||
}
|
||||
}
|
||||
want := []dropped{{nodebuildgo.Seat, "n7", "a1"}, {nodebuildgo.Seat, "", "a2"}}
|
||||
if !slices.Equal(drops, want) || !slices.Equal(unbound, []string{"a1", "a2"}) {
|
||||
t.Errorf("dropped %+v and unbound %v, want %+v (the mesh-builds step only ended)", drops, unbound, want)
|
||||
if !slices.Equal(drops, want) {
|
||||
t.Errorf("dropped %+v, want %+v (the mesh-builds step only ended)", drops, want)
|
||||
}
|
||||
if n := len(*said); n != 5 || (*said)[4] != seat.EventPipelineEnded+".app.main.c1" {
|
||||
t.Errorf("said %v, want four step-ended then pipeline-ended", *said)
|
||||
}
|
||||
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop, unbind); n != 0 {
|
||||
if n, _ := h.Supersede(newer("main.7.app", 7, "src2:cc", "", nil), drop); n != 0 {
|
||||
t.Errorf("a superseded pipeline was superseded again: %d", n)
|
||||
}
|
||||
|
||||
@@ -82,10 +80,10 @@ func TestSupersede(t *testing.T) {
|
||||
pub := memStore{"main.5.lib": {p: seat.Pipeline{Pipeline: "main.5.lib", Branch: "main", Source: 5, Module: "lib", Fingerprint: "src2:aa",
|
||||
Outcome: seat.PipelineOutcomeRunning, Published: &seat.Published{Artifact: "core:pkg", Version: "v1"}}, rev: 1}}
|
||||
h.Store = pub
|
||||
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop, unbind); n != 0 {
|
||||
if n, _ := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v1"}), drop); n != 0 {
|
||||
t.Errorf("a publish of the same version superseded %d", n)
|
||||
}
|
||||
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop, unbind); n != 1 || err != nil {
|
||||
if n, err := h.Supersede(newer("main.6.app", 6, "src2:aa", "", &seat.Published{Artifact: "core:pkg", Version: "v2"}), drop); n != 1 || err != nil {
|
||||
t.Fatalf("a newer publish superseded %d (%v)", n, err)
|
||||
}
|
||||
if p, _, _ := pub.Get("main.5.lib"); p.Outcome != seat.PipelineOutcomeSuperseded || !strings.Contains(p.Why, "v2") {
|
||||
@@ -105,9 +103,6 @@ func TestBoundDrop(t *testing.T) {
|
||||
if _, _, ok := ParseDropBound("step-ab"); ok {
|
||||
t.Error("an ask's bound was read as a drop's")
|
||||
}
|
||||
if IsAskKey(DropBoundKey("s", "step-ab")) {
|
||||
t.Error("a drop's bound was taken for an ask's")
|
||||
}
|
||||
err := BoundDrop(func(string, string, string, string) error { return nil }, arm, done)(nodebuildgo.Seat, "n1", "step-ab", "why")
|
||||
if err != nil || len(armed) != 1 || armed[0] != "drop."+nodebuildgo.Seat+".step-ab" || len(answered) != 1 {
|
||||
t.Errorf("an answered drop: %v, armed %v, answered %v", err, armed, answered)
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// TestRuns is the runner's state test-runs (design 47 G §725): one key per test run of a candidate, level and tested
|
||||
// packages, written create-only by the step that asks the run and changed by compare-and-set.
|
||||
type TestRuns interface {
|
||||
// Create writes key when it is absent; ErrExists when it is written already.
|
||||
Create(key string, r Run) error
|
||||
// Get reads key and its revision; ErrNoRun when it is absent.
|
||||
Get(key string) (Run, uint64, error)
|
||||
// Update writes key at the revision read; ErrChanged when another write came between.
|
||||
Update(key string, r Run, revision uint64) (uint64, error)
|
||||
// Delete deletes key at the revision read; ErrChanged when another write came between, a key gone is no fault. No
|
||||
// write of a run puts a value judged from an earlier read over a later one (the kind of the stale write).
|
||||
Delete(key string, revision uint64) error
|
||||
}
|
||||
|
||||
// Run is a key's value in test-runs (§725): the test run, its readers each marked whether a rerun's new run moves it and by
|
||||
// whom (A-470d), so the marks are in the key and a restart takes them up.
|
||||
type Run struct {
|
||||
seat.TestRun
|
||||
Readers []RunReader `json:"readers,omitempty"`
|
||||
}
|
||||
|
||||
// RunReader is a step reading a test run, its pipeline key and artifact (§725), marked rerun with the caller of the rerun
|
||||
// whose new run moves it (A-470d).
|
||||
type RunReader struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Artifact string `json:"artifact"`
|
||||
Rerun bool `json:"rerun,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// Of is the reader as the seat names it.
|
||||
func (r RunReader) Of() seat.TestRunReader {
|
||||
return seat.TestRunReader{Pipeline: r.Pipeline, Artifact: r.Artifact}
|
||||
}
|
||||
|
||||
// Answer is the run as the seat's read answers it, its readers named by pipeline and artifact.
|
||||
func (r Run) Answer() seat.TestRun {
|
||||
t := r.TestRun
|
||||
t.Readers = nil
|
||||
for _, o := range r.Readers {
|
||||
t.Readers = append(t.Readers, o.Of())
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// ErrNoRun is a Get of a test-runs key that is not written.
|
||||
var ErrNoRun = errors.New("no test run under the key")
|
||||
|
||||
// CandidateOf is a pipeline's candidate in A-226's form (§724): `run-<id>` for a run on request, else `<branch>.<revision>`
|
||||
// of the catalog graph revision it was placed from, built by the node-engine seat's client as the assigner builds it. A
|
||||
// pipeline naming neither has no candidate, and its steps share no run.
|
||||
func CandidateOf(p seat.Pipeline) (string, bool) {
|
||||
var key string
|
||||
var err error
|
||||
if p.Run != "" {
|
||||
key, err = nec.RunKey(p.Run)
|
||||
} else {
|
||||
key, err = nec.RevisionKey(p.Branch, p.Revision)
|
||||
}
|
||||
return key, err == nil
|
||||
}
|
||||
|
||||
// ShareDigest is a test step's share digest (design 47 G §724, A-469c): the sha256 of the module's build-input fingerprint,
|
||||
// the level, the step's status in the policy and the step's fetch (its directory and its sorted paths), what the runner
|
||||
// holds, so two artifacts with one fetch meet and two with different fetches do not.
|
||||
func ShareDigest(fingerprint, level, status string, fetch nodebuildgo.Fetch) string {
|
||||
paths := append([]string(nil), fetch.Paths...)
|
||||
sort.Strings(paths)
|
||||
h := sha256.New()
|
||||
for _, part := range []string{fingerprint, level, status, fetch.Dir} {
|
||||
h.Write([]byte(part + "\n"))
|
||||
}
|
||||
h.Write([]byte(strings.Join(paths, "\n")))
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// TestRunKey is a test run's key in test-runs (§725): `<candidate key>.<level>.<digest>`, the level and the digest each
|
||||
// escaped as a key's name, the candidate already in its key form.
|
||||
func TestRunKey(candidate, level, digest string) string {
|
||||
return candidate + "." + keys.EscapeName(level) + "." + keys.EscapeName(digest)
|
||||
}
|
||||
|
||||
// sharedRun is the one run a step's Shared names, empty when it names none.
|
||||
func sharedRun(s seat.StepResult) string {
|
||||
if len(s.Shared) == 0 {
|
||||
return ""
|
||||
}
|
||||
return s.Shared[0].Run
|
||||
}
|
||||
|
||||
// readsRun says whether step s of a pipeline reads run as one step that does not ask it: it names the run and keeps no ask
|
||||
// of its own (A-469m).
|
||||
func readsRun(s seat.StepResult, run string) bool {
|
||||
return run != "" && sharedRun(s) == run && s.Ask == ""
|
||||
}
|
||||
|
||||
// addReaders adds each reader not in r's readers yet, in order.
|
||||
func addReaders(r *Run, readers []seat.TestRunReader) {
|
||||
for _, n := range readers {
|
||||
have := false
|
||||
for _, o := range r.Readers {
|
||||
have = have || o.Of() == n
|
||||
}
|
||||
if !have {
|
||||
r.Readers = append(r.Readers, RunReader{Pipeline: n.Pipeline, Artifact: n.Artifact})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// askerOf is the step that asked the run under ask: the step of a pipeline the runner keeps whose own ask it is.
|
||||
func askerOf(store Updater, ask string) (*seat.SharedAskedBy, error) {
|
||||
if store == nil || ask == "" {
|
||||
return nil, nil
|
||||
}
|
||||
pkeys, err := store.Keys("*.*.*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if i := stepOfAsk(p, ask); i >= 0 {
|
||||
return &seat.SharedAskedBy{Pipeline: key, Artifact: p.Steps[i].Artifact}, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// share writes the test run of each ask that asks one (§724, §725): the step that writes the key create-only asks the run;
|
||||
// one that finds the key written by another step reads it, its pipeline's steps of that run then keeping no ask of their
|
||||
// own and naming the run's ask and its asker (A-469m), and its ask is not put. A run ended `could not run` is never read:
|
||||
// it is replaced by compare-and-set and asked again. A key written by this very ask (a start asked again) asks it again,
|
||||
// kept once by its id. It answers the asks still to put.
|
||||
func (s *Starter) share(key string, asks []Asked) []Asked {
|
||||
if s.TestRuns == nil {
|
||||
return asks
|
||||
}
|
||||
var out []Asked
|
||||
joined := map[string]Run{}
|
||||
for _, a := range asks {
|
||||
if a.Run == "" {
|
||||
out = append(out, a)
|
||||
continue
|
||||
}
|
||||
run, ok, err := s.claimRun(a)
|
||||
var full *fullErr
|
||||
switch {
|
||||
case errors.As(err, &full):
|
||||
// At the cap no run is asked: the steps end could not run, naming the fill (§725, A-469d).
|
||||
s.logf("the test run %s of %s is not asked: %v", a.Run, key, err)
|
||||
if s.EndReaders != nil {
|
||||
if err := s.EndReaders(a.Run, a.Readers, seat.TestRun{Outcome: seat.StepResultOutcomeCouldNotRun, Said: err.Error()}); err != nil {
|
||||
s.logf("the steps of the test run %s, not asked, are not ended: %v", a.Run, err)
|
||||
}
|
||||
}
|
||||
case err != nil:
|
||||
// A run that cannot be written or read is asked as the step's own, said: slower, never lost.
|
||||
s.logf("the test run %s of %s is not written, so the step is asked on its own: %v", a.Run, key, err)
|
||||
out = append(out, a)
|
||||
case ok:
|
||||
out = append(out, a)
|
||||
default:
|
||||
joined[a.Run] = run
|
||||
}
|
||||
}
|
||||
if len(joined) > 0 {
|
||||
s.readOthers(key, joined)
|
||||
// A run that ended within the candidate is read at once (§724): its readers end with its result.
|
||||
for run, r := range joined {
|
||||
if r.Ended == nil || s.EndReaders == nil {
|
||||
continue
|
||||
}
|
||||
var readers []seat.TestRunReader
|
||||
for _, a := range asks {
|
||||
if a.Run == run {
|
||||
readers = a.Readers
|
||||
}
|
||||
}
|
||||
if err := s.EndReaders(run, readers, r.TestRun); err != nil {
|
||||
s.logf("the steps of %s reading the ended test run %s are not ended: %v", key, run, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fullErr is a run not written because test-runs is at its cap (A-469d), naming the fill.
|
||||
type fullErr struct{ fill seat.Fill }
|
||||
|
||||
func (e *fullErr) Error() string {
|
||||
return fmt.Sprintf("the state %s holds %d of its %d bytes, so no test run is asked (test-runs-cap-mib)", e.fill.Store, e.fill.Bytes, e.fill.Cap)
|
||||
}
|
||||
|
||||
// TestRunsFill is test-runs' fill against its cap setting test-runs-cap-mib, the cap that refuses a new run (A-469d, A-470i),
|
||||
// never the manifest's ceiling.
|
||||
func (s *Starter) TestRunsFill() (seat.Fill, error) {
|
||||
if s.RunsFill == nil {
|
||||
return seat.Fill{}, errors.New("the runner reads no fill of test-runs")
|
||||
}
|
||||
f, err := s.RunsFill()
|
||||
if err != nil {
|
||||
return seat.Fill{}, err
|
||||
}
|
||||
f.Cap = s.Runs.CapBytes
|
||||
if f.Cap <= 0 {
|
||||
f.Cap = DefaultRunSettings.CapBytes
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// atCap is the error of test-runs at its cap setting, nil below it or when its fill is not read (said).
|
||||
func (s *Starter) atCap() error {
|
||||
if s.RunsFill == nil {
|
||||
return nil
|
||||
}
|
||||
f, err := s.TestRunsFill()
|
||||
if err != nil {
|
||||
s.logf("the fill of test-runs is not read, so its cap is not judged: %v", err)
|
||||
return nil
|
||||
}
|
||||
if f.Bytes >= f.Cap {
|
||||
return &fullErr{fill: f}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// started writes a new run and its untaken bound (§725, A-470).
|
||||
func (s *Starter) started(a Asked, write func() error) error {
|
||||
if err := s.atCap(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := write(); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Clock != nil {
|
||||
untaken := s.Runs.Untaken
|
||||
if untaken <= 0 {
|
||||
untaken = DefaultRunSettings.Untaken
|
||||
}
|
||||
if err := armBound(s.Clock, s.TestRuns, UntakenPrefix+a.Run, a.Run, a.ID, untaken); err != nil {
|
||||
s.logf("the untaken bound of the test run %s is not written: %v", a.Run, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// claimRun writes a's test run create-only; ok says the ask is put (the key is a's own), else run is the run it reads.
|
||||
func (s *Starter) claimRun(a Asked) (Run, bool, error) {
|
||||
now := s.now()
|
||||
value := Run{TestRun: seat.TestRun{Level: a.Level, Digest: a.Digest, Seat: a.Seat, Ask: a.ID, Started: &now}}
|
||||
addReaders(&value, a.Readers)
|
||||
for {
|
||||
err := s.started(a, func() error { return s.TestRuns.Create(a.Run, value) })
|
||||
if err == nil {
|
||||
return value, true, nil
|
||||
}
|
||||
var full *fullErr
|
||||
if errors.As(err, &full) {
|
||||
return Run{}, false, err
|
||||
}
|
||||
if !errors.Is(err, ErrExists) {
|
||||
return Run{}, false, err
|
||||
}
|
||||
r, rev, err := s.TestRuns.Get(a.Run)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
continue // deleted meanwhile: written again
|
||||
}
|
||||
if err != nil {
|
||||
return Run{}, false, err
|
||||
}
|
||||
if r.Ask == a.ID {
|
||||
return r, true, nil
|
||||
}
|
||||
if r.Ended != nil && r.Outcome == seat.StepResultOutcomeCouldNotRun {
|
||||
err := s.started(a, func() error {
|
||||
_, err := s.TestRuns.Update(a.Run, value, rev)
|
||||
return err
|
||||
})
|
||||
if errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Run{}, false, err
|
||||
}
|
||||
return value, true, nil
|
||||
}
|
||||
addReaders(&r, a.Readers)
|
||||
if _, err := s.TestRuns.Update(a.Run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return Run{}, false, err
|
||||
}
|
||||
return r, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// readOthers writes pipeline key's steps that read another step's run: no ask of their own, the run's ask and its asker
|
||||
// in Shared (A-469m, A-469q).
|
||||
func (s *Starter) readOthers(key string, joined map[string]Run) {
|
||||
if s.Store == nil {
|
||||
return
|
||||
}
|
||||
by := map[string]*seat.SharedAskedBy{}
|
||||
for run, r := range joined {
|
||||
asker, err := askerOf(s.Store, r.Ask)
|
||||
if err != nil {
|
||||
s.logf("the step that asked the test run %s is not read: %v", run, err)
|
||||
}
|
||||
by[run] = asker
|
||||
}
|
||||
for {
|
||||
p, rev, err := s.Store.Get(key)
|
||||
if err != nil {
|
||||
s.logf("the pipeline %s is not read, so its steps reading other runs are not written: %v", key, err)
|
||||
return
|
||||
}
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
r, ok := joined[sharedRun(*st)]
|
||||
if !ok || !open(*st) {
|
||||
continue
|
||||
}
|
||||
if r.Node != "" {
|
||||
st.Outcome, st.On = seat.StepResultOutcomeRunning, "" // its run's ask is taken already (A-469b)
|
||||
}
|
||||
st.Ask = ""
|
||||
st.Shared = []seat.Shared{{Run: sharedRun(*st), Ask: r.Ask, AskedBy: by[sharedRun(*st)]}}
|
||||
}
|
||||
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
s.logf("the pipeline %s's steps reading other runs are not written: %v", key, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// joinRun adds reader to run's readers by compare-and-set.
|
||||
func (s *Starter) joinRun(run string, reader seat.TestRunReader) {
|
||||
if s.TestRuns == nil {
|
||||
return
|
||||
}
|
||||
for {
|
||||
r, rev, err := s.TestRuns.Get(run)
|
||||
if err != nil {
|
||||
s.logf("the test run %s is not read, so %s does not join it: %v", run, reader.Pipeline, err)
|
||||
return
|
||||
}
|
||||
addReaders(&r, []seat.TestRunReader{reader})
|
||||
if _, err := s.TestRuns.Update(run, r, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
s.logf("%s does not join the test run %s: %v", reader.Pipeline, run, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// RunOutput is a test run's output object in test-runs-objects (§725): the packages the run ran with their digests, the
|
||||
// passing ones it kept and the carried ones it skipped, and the build seat's output.
|
||||
type RunOutput struct {
|
||||
Packages []seat.TestRunPackage `json:"packages"`
|
||||
Output string `json:"output,omitempty"`
|
||||
}
|
||||
|
||||
// Gone is what a run whose output object is gone answers in place of its packages (§725, §733).
|
||||
const Gone = "the run's output is gone"
|
||||
|
||||
// RunsRead answers the seat verb test-runs (design 47 G §728; A-469u) from the runner's state test-runs and its object
|
||||
// store: each run with its packages read from its output object, and the state's fill.
|
||||
type RunsRead struct {
|
||||
Runs Kept
|
||||
// Object reads an object of test-runs-objects; ErrNoRun when it is gone.
|
||||
Object func(name string) ([]byte, error)
|
||||
Fill func() (seat.Fill, error)
|
||||
}
|
||||
|
||||
// TestRuns answers each test run of a candidate, a level, a digest or a module, or every one.
|
||||
func (r RunsRead) TestRuns(in seat.TestRunsInput) (seat.TestRunsAnswer, error) {
|
||||
all, err := r.Runs.Keys()
|
||||
if err != nil {
|
||||
return seat.TestRunsAnswer{}, fmt.Errorf("the test runs are not listed: %w", err)
|
||||
}
|
||||
sort.Strings(all)
|
||||
a := seat.TestRunsAnswer{Runs: []seat.TestRun{}}
|
||||
for _, key := range all {
|
||||
if IsRunBound(key) || (in.Candidate != "" && !strings.HasPrefix(key, in.Candidate+".")) {
|
||||
continue
|
||||
}
|
||||
raw, err := r.Runs.Value(key)
|
||||
if err != nil || raw == nil {
|
||||
continue
|
||||
}
|
||||
var run Run
|
||||
if err := json.Unmarshal(raw, &run); err != nil {
|
||||
continue
|
||||
}
|
||||
if (in.Level != "" && run.Level != in.Level) || (in.Digest != "" && run.Digest != in.Digest) || (in.Module != "" && !readsModule(run, in.Module)) {
|
||||
continue
|
||||
}
|
||||
t := run.Answer()
|
||||
t.Key = key
|
||||
r.packages(&t)
|
||||
a.Runs = append(a.Runs, t)
|
||||
}
|
||||
if r.Fill != nil {
|
||||
f, err := r.Fill()
|
||||
if err != nil {
|
||||
return seat.TestRunsAnswer{}, fmt.Errorf("the fill of test-runs cannot be read: %w", err)
|
||||
}
|
||||
a.Fill = f
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// packages names t's packages from its output object, or says it gone (§725, A-466v).
|
||||
func (r RunsRead) packages(t *seat.TestRun) {
|
||||
if t.Output == "" || r.Object == nil {
|
||||
return
|
||||
}
|
||||
data, err := r.Object(t.Output)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
t.Gone = Gone
|
||||
return
|
||||
}
|
||||
var out RunOutput
|
||||
if err == nil {
|
||||
err = json.Unmarshal(data, &out)
|
||||
}
|
||||
if err != nil {
|
||||
t.Gone = "the run's output is not read: " + err.Error()
|
||||
return
|
||||
}
|
||||
t.Packages = out.Packages
|
||||
}
|
||||
|
||||
// readsModule says whether a reader of run is a step of module, its artifact `<module>:<name>`.
|
||||
func readsModule(run Run, module string) bool {
|
||||
for _, o := range run.Readers {
|
||||
if strings.HasPrefix(o.Artifact, module+":") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestRunsDrop is the seat verb test-runs-drop (design 47 G §728; A-469g, A-469v), answered by the runner's process: the
|
||||
// hand act recorded first, the run's ask dropped, each reader ended could not run naming the drop, then the run's output
|
||||
// object and its key deleted. It answers the run deleted and the readers it ended.
|
||||
func (v Verbs) TestRunsDrop(caller string, in seat.TestRunsDropInput) (seat.TestRunsDropAnswer, error) {
|
||||
h := v.Hearer
|
||||
if h.TestRuns == nil {
|
||||
return seat.TestRunsDropAnswer{}, errors.New("the runner keeps no test runs")
|
||||
}
|
||||
r, _, err := h.TestRuns.Get(in.Key)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("no test run %q in test-runs", in.Key)
|
||||
}
|
||||
if err != nil {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read: %w", in.Key, err)
|
||||
}
|
||||
if err := v.record(seat.VerbTestRunsDrop, "dropped the test run "+in.Key, in.Why); err != nil {
|
||||
return seat.TestRunsDropAnswer{}, err
|
||||
}
|
||||
why := fmt.Sprintf("the test run %s was dropped by %s: %s", in.Key, caller, in.Why)
|
||||
var ended []seat.TestRunReader
|
||||
if r.Ended == nil {
|
||||
if h.DropRun != nil && r.Ask != "" {
|
||||
if err := h.DropRun(r.Seat, r.Node, r.Ask, why); err != nil {
|
||||
h.logf("the drop of ask %s of the dropped test run %s is not answered: %v", r.Ask, in.Key, err)
|
||||
}
|
||||
}
|
||||
if err := h.EndRun(in.Key, seat.TestRun{Ask: r.Ask, Outcome: seat.StepResultOutcomeCouldNotRun, Said: why}, nil); err != nil {
|
||||
return seat.TestRunsDropAnswer{}, err
|
||||
}
|
||||
for _, o := range r.Readers {
|
||||
ended = append(ended, o.Of())
|
||||
}
|
||||
}
|
||||
if h.RunOutputs != nil {
|
||||
// The run's object is named by its key (endRunOf); one the run named otherwise is deleted too.
|
||||
for _, name := range []string{in.Key, r.Output} {
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if err := h.RunOutputs.Delete(name); err != nil {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the output object %s of the test run is not deleted, so its key is kept: %w", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The key is deleted at the revision of the run dropped: a run a rerun asked under it since is not deleted with it.
|
||||
for {
|
||||
now, rev, err := h.TestRuns.Get(in.Key)
|
||||
if errors.Is(err, ErrNoRun) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not read again, so it is not deleted: %w", in.Key, err)
|
||||
}
|
||||
if now.Ask != r.Ask {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the key %s holds the run of ask %s, asked since the run of ask %s was dropped: it is not deleted", in.Key, now.Ask, r.Ask)
|
||||
}
|
||||
if err := h.TestRuns.Delete(in.Key, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return seat.TestRunsDropAnswer{}, fmt.Errorf("the test run %s is not deleted: %w", in.Key, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
if h.Clock != nil {
|
||||
for _, k := range []string{BoundPrefix + in.Key, UntakenPrefix + in.Key} {
|
||||
_ = h.Clock.Unbound(k)
|
||||
}
|
||||
}
|
||||
gone := r.Answer()
|
||||
gone.Key = in.Key
|
||||
now := h.now()
|
||||
if gone.Ended == nil {
|
||||
gone.Ended, gone.Outcome, gone.Said = &now, seat.StepResultOutcomeCouldNotRun, why
|
||||
}
|
||||
return seat.TestRunsDropAnswer{Run: gone, Ended: ended}, nil
|
||||
}
|
||||
@@ -0,0 +1,672 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// clockMem is the bound keys of test-runs in memory, each with the time to live it was last written with, the ask it holds
|
||||
// and its revision.
|
||||
type clockMem struct {
|
||||
runs testRunsMem
|
||||
bound map[string]time.Duration
|
||||
asks map[string]string
|
||||
revs map[string]uint64
|
||||
rev uint64
|
||||
// before is called once at the next Bound, a write that comes between its caller's read and its write.
|
||||
before func()
|
||||
}
|
||||
|
||||
func (c *clockMem) Bound(key, ask string, ttl time.Duration) (uint64, error) {
|
||||
if b := c.before; b != nil {
|
||||
c.before = nil
|
||||
b()
|
||||
}
|
||||
if c.asks == nil {
|
||||
c.asks, c.revs = map[string]string{}, map[string]uint64{}
|
||||
}
|
||||
c.rev++
|
||||
c.bound[key], c.asks[key], c.revs[key] = ttl, ask, c.rev
|
||||
return c.rev, nil
|
||||
}
|
||||
func (c *clockMem) Bounded(key string) (string, uint64, bool, error) {
|
||||
_, ok := c.bound[key]
|
||||
return c.asks[key], c.revs[key], ok, nil
|
||||
}
|
||||
func (c *clockMem) Unbound(key string) error { delete(c.bound, key); return nil }
|
||||
func (c *clockMem) UnboundAt(key string, rev uint64) error {
|
||||
if c.revs[key] == rev {
|
||||
delete(c.bound, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (c *clockMem) Has(key string) (bool, error) { _, ok := c.bound[key]; return ok, nil }
|
||||
func (c *clockMem) Keys() ([]string, error) {
|
||||
var out []string
|
||||
for k := range c.runs {
|
||||
out = append(out, k)
|
||||
}
|
||||
for k := range c.bound {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type objectsMem map[string][]byte
|
||||
|
||||
func (o objectsMem) Put(name string, data []byte) error { o[name] = data; return nil }
|
||||
func (o objectsMem) Get(name string) ([]byte, error) {
|
||||
if d, ok := o[name]; ok {
|
||||
return d, nil
|
||||
}
|
||||
return nil, ErrNoRun
|
||||
}
|
||||
func (o objectsMem) Delete(name string) error { delete(o, name); return nil }
|
||||
|
||||
// runFixture is a starter and a hearer over one pipelines store and one test-runs, the module's program and tools testing
|
||||
// one directory.
|
||||
type runFixture struct {
|
||||
store memStore
|
||||
runs testRunsMem
|
||||
clock *clockMem
|
||||
objects objectsMem
|
||||
asks *stepAsks
|
||||
drops []string
|
||||
said []seat.StepEvent
|
||||
logged []string
|
||||
fill int64
|
||||
s *Starter
|
||||
h *Hearer
|
||||
}
|
||||
|
||||
func newRunFixture() *runFixture {
|
||||
f := &runFixture{store: memStore{}, runs: testRunsMem{}, objects: objectsMem{}, asks: &stepAsks{}}
|
||||
f.clock = &clockMem{runs: f.runs, bound: map[string]time.Duration{}}
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}}
|
||||
say := func(_ string, body any) error {
|
||||
if e, ok := body.(seat.StepEvent); ok {
|
||||
f.said = append(f.said, e)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
f.h = &Hearer{Store: f.store, Say: say, TestRuns: f.runs, Clock: f.clock, RunOutputs: f.objects,
|
||||
Logf: func(format string, a ...any) { f.logged = append(f.logged, fmt.Sprintf(format, a...)) },
|
||||
DropRun: func(seatName, node, ask, why string) error {
|
||||
f.drops = append(f.drops, seatName+" "+node+" "+ask)
|
||||
return nil
|
||||
}}
|
||||
f.s = &Starter{Pipelines: memCreated{f.store}, Store: f.store, TestRuns: f.runs, Clock: f.clock, Say: say, Steps: f.asks,
|
||||
Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }, EndReaders: f.h.EndReaders,
|
||||
RunsFill: func() (seat.Fill, error) { return seat.Fill{Store: "test-runs", Bytes: f.fill}, nil }, Logf: f.h.Logf}
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *runFixture) start(t *testing.T, source, revision int64) seat.Pipeline {
|
||||
t.Helper()
|
||||
entries := map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}},
|
||||
}
|
||||
if ok, err := f.s.Start(Begin{Branch: "main", Source: source, Module: "app", Commit: "c1", Fp: "src2:aa", Revision: revision, Entries: entries}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
p, _, _ := f.store.Get(Key("main", source, "app"))
|
||||
return p
|
||||
}
|
||||
|
||||
func (f *runFixture) pipeline(p seat.Pipeline) seat.Pipeline {
|
||||
q, _, _ := f.store.Get(p.Pipeline)
|
||||
return q
|
||||
}
|
||||
|
||||
// The test runs' clocks and cap (design 47 G §725; A-466n, A-466p, A-466s, A-468b, A-469d, A-470, A-470a): a run's untaken
|
||||
// bound is written at its start, its clock at ask-taken and again at each step-progress; a clock's expiry ends the run and
|
||||
// every reader could not run and drops its ask, a later step-ended ignored and journaled; an untaken bound of a taken run
|
||||
// ends nothing; a run whose bound passed while the runner was away ends at its start; at the cap no run is asked; an ended
|
||||
// run is aged with its output object in the step log's order.
|
||||
func TestATestRunHasOneClockACapAndAnAge(t *testing.T) {
|
||||
t.Run("clock", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
f.h.Runs = RunSettings{StepBound: 30 * time.Minute, Untaken: 24 * time.Hour}
|
||||
f.s.Runs = f.h.Runs
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
if f.clock.bound[UntakenPrefix+run] != 24*time.Hour || len(f.clock.bound) != 3 {
|
||||
t.Fatalf("bounds at the start = %v, want the three runs' untaken bounds of 24h", f.clock.bound)
|
||||
}
|
||||
// A-469b: the run's steps wait for a build node until the holder's ask-taken, then run, each said started again.
|
||||
if unit.Outcome != seat.StepResultOutcomeWaitingForABuildNode || stepNamed(p, "app:tools", "unit").Outcome != seat.StepResultOutcomeWaitingForABuildNode {
|
||||
t.Fatalf("the unit steps before ask-taken = %s, %s", unit.Outcome, stepNamed(p, "app:tools", "unit").Outcome)
|
||||
}
|
||||
n := len(f.said)
|
||||
if err := f.h.RunTaken(unit.Ask, "ace"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
if s := stepNamed(f.pipeline(p), art, "unit"); s.Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Errorf("the %s unit step after ask-taken is %s, want running", art, s.Outcome)
|
||||
}
|
||||
}
|
||||
if len(f.said)-n != 2 || f.said[n].Result.Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Errorf("ask-taken said %d step events, want the two unit steps started running", len(f.said)-n)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Node != "ace" || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
|
||||
t.Fatalf("after ask-taken the run = %+v, bounds %v; want its node and its clock of 30m", r, f.clock.bound)
|
||||
}
|
||||
if _, ok := f.clock.bound[UntakenPrefix+run]; ok {
|
||||
t.Error("the untaken bound of a taken run is kept")
|
||||
}
|
||||
// An untaken bound of a taken run ends nothing; progress begins the clock again.
|
||||
if err := f.h.RunExpired(UntakenPrefix + run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delete(f.clock.bound, BoundPrefix+run)
|
||||
if err := f.h.RunProgress(unit.Ask); err != nil || f.clock.bound[BoundPrefix+run] != 30*time.Minute {
|
||||
t.Fatalf("progress = %v, bounds %v: want the clock begun again", err, f.clock.bound)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Ended != nil {
|
||||
t.Fatal("a run is ended by its untaken bound after it was taken")
|
||||
}
|
||||
// An expiry heard once the bound is armed again ends nothing (A-470o); the bound holds the run's ask (A-470n).
|
||||
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Ended != nil || f.clock.asks[BoundPrefix+run] != unit.Ask {
|
||||
t.Fatalf("an expiry of a bound armed again ended the run %+v, or the bound holds ask %q", r, f.clock.asks[BoundPrefix+run])
|
||||
}
|
||||
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
|
||||
if err := f.h.RunExpired(BoundPrefix + run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, _, _ := f.runs.Get(run)
|
||||
if r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, unit.Ask) || !strings.Contains(r.Said, "30m") {
|
||||
t.Errorf("the run whose clock ended = %+v, want could not run naming its ask and the bound", r)
|
||||
}
|
||||
if len(f.drops) != 1 || f.drops[0] != "node-build-go ace "+unit.Ask {
|
||||
t.Errorf("drops = %v, want the run's ask dropped of ace", f.drops)
|
||||
}
|
||||
q := f.pipeline(p)
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != r.Said {
|
||||
t.Errorf("the %s unit step = %+v, want could not run with the run", art, s)
|
||||
}
|
||||
}
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := stepNamed(f.pipeline(p), "app:program", "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun {
|
||||
t.Errorf("a step-ended after the run ended moved the step to %s", s.Outcome)
|
||||
}
|
||||
if !strings.Contains(strings.Join(f.logged, "\n"), "the step-ended of ask "+unit.Ask+" is ignored: its test run "+run) {
|
||||
t.Errorf("the ignored step-ended is not journaled: %q", f.logged)
|
||||
}
|
||||
// An untaken run whose bound passed while the runner was away ends at the start.
|
||||
lint := stepNamed(p, "app:program", "lint").Shared[0].Run
|
||||
delete(f.clock.bound, UntakenPrefix+lint)
|
||||
if n, err := f.h.TakeUp(); n != 1 || err != nil {
|
||||
t.Fatalf("take-up ended %d, %v; want the lint run", n, err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(lint); r.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(r.Said, "run-untaken-bound") {
|
||||
t.Errorf("the lint run taken up = %+v", r)
|
||||
}
|
||||
})
|
||||
t.Run("cap", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
f.s.Runs.CapBytes, f.fill = 1<<20, 1<<20
|
||||
p := f.start(t, 4, 9)
|
||||
for _, a := range f.asks.ids {
|
||||
if a == stepNamed(p, "app:program", "unit").Ask {
|
||||
t.Error("a unit run is asked at the cap")
|
||||
}
|
||||
}
|
||||
q := f.pipeline(p)
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
if s := stepNamed(q, art, "unit"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "1048576 of its 1048576 bytes") {
|
||||
t.Errorf("the %s unit step at the cap = %+v, want could not run naming the fill", art, s)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("age", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed, Output: "out-1"}, nil)
|
||||
f.objects["out-1"] = []byte("ok")
|
||||
now := agedNow
|
||||
a := &Ager{Runs: f.runs, RunOutputs: f.objects.Delete, RunsKept: 7 * 24 * time.Hour, Pipelines: agedDeleter{f.store},
|
||||
Aged: agedState{}, Now: func() time.Time { return now },
|
||||
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
|
||||
if from > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
|
||||
What: seat.LogEntryWhatEnded, At: agedNow.Add(-6 * 24 * time.Hour)}}}, nil
|
||||
}}
|
||||
a.Age()
|
||||
if _, _, err := f.runs.Get(run); err != nil {
|
||||
t.Fatal("a run is aged before test-runs-kept passed")
|
||||
}
|
||||
now = now.Add(2 * 24 * time.Hour)
|
||||
a.Age()
|
||||
if _, _, err := f.runs.Get(run); err != ErrNoRun || len(f.objects) != 0 {
|
||||
t.Errorf("the run past test-runs-kept is %v, objects %v; want both deleted", err, f.objects)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// agedDeleter is a memStore with the delete the ager makes.
|
||||
type agedDeleter struct{ memStore }
|
||||
|
||||
func (d agedDeleter) Delete(key string) error { delete(d.memStore, key); return nil }
|
||||
|
||||
// How a test run ends and is said (design 47 G §726; A-466i, A-468d, A-469m, A-469z): the run's ask's end ends every step
|
||||
// reading it with its outcome, counts and shared, its output kept in test-runs-objects; a pipeline counts the run's tests
|
||||
// once; a step superseded leaves the run to its other readers, and the last to leave drops its ask and ends it could not
|
||||
// run, no reader left; a composition step's end keeps the runs mesh-wants names.
|
||||
func TestATestRunsEndMovesEveryReaderAndIsCountedOnce(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
a := f.start(t, 4, 9)
|
||||
b := f.start(t, 6, 9) // the same candidate: it reads a's runs
|
||||
unit := stepNamed(a, "app:program", "unit")
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: a.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed",
|
||||
Said: "TestX failed", Output: "--- FAIL: TestX", Tests: &nodebuildgo.TestCounts{Unit: 5}, Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range []seat.Pipeline{f.pipeline(a), f.pipeline(b)} {
|
||||
for _, art := range []string{"app:program", "app:tools"} {
|
||||
s := stepNamed(p, art, "unit")
|
||||
if s.Outcome != seat.StepResultOutcomeFailed || s.Why != "TestX failed" || s.Tests == nil || s.Tests.Unit != 5 ||
|
||||
len(s.Shared) != 1 || s.Shared[0].Run != unit.Shared[0].Run || s.Shared[0].Ask != unit.Ask {
|
||||
t.Errorf("%s's %s unit step = %+v, want failed with the run's result and shared", p.Pipeline, art, s)
|
||||
}
|
||||
}
|
||||
if c := testCounts(p.Steps); c == nil || c.Unit != 5 {
|
||||
t.Errorf("%s counts %+v, want the run's 5 unit tests once", p.Pipeline, c)
|
||||
}
|
||||
}
|
||||
if r, _, _ := f.runs.Get(unit.Shared[0].Run); r.Outcome != "failed" || r.Output != unit.Shared[0].Run || !strings.Contains(string(f.objects[r.Output]), `"output":"--- FAIL: TestX"`) {
|
||||
t.Errorf("the ended run = %+v, objects %v; want its output kept under its key", r, f.objects)
|
||||
}
|
||||
saidReaders := 0
|
||||
for _, e := range f.said {
|
||||
if e.Result.Step == "unit" && e.Result.Outcome == "failed" {
|
||||
saidReaders++
|
||||
}
|
||||
}
|
||||
if saidReaders != 4 {
|
||||
t.Errorf("said %d unit step-ended, want one per reading step", saidReaders)
|
||||
}
|
||||
// a, superseded, leaves the lint run to b's readers; b, superseded after, leaves none: the ask is dropped and the run ends.
|
||||
lint := stepNamed(a, "app:program", "lint")
|
||||
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.5.app", Branch: "main", Source: 5, Module: "app", Commit: "c2", Fingerprint: "src2:bb"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Ended != nil || len(r.Readers) != 2 || len(f.drops) != 0 {
|
||||
t.Fatalf("after a left the lint run = %+v, drops %v; want it running for b's two readers", r, f.drops)
|
||||
}
|
||||
if _, err := f.h.Supersede(seat.Pipeline{Pipeline: "main.7.app", Branch: "main", Source: 7, Module: "app", Commit: "c3", Fingerprint: "src2:cc"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(lint.Shared[0].Run); r.Outcome != seat.StepResultOutcomeCouldNotRun || r.Said != NoReaderLeft || len(f.drops) != 2 || f.drops[0] != "node-build-go "+lint.Ask {
|
||||
t.Errorf("after the last reader left the lint run = %+v, drops %v; want it ended, no reader left, its ask and the integration run's dropped", r, f.drops)
|
||||
}
|
||||
// A composition step's end keeps the runs mesh-wants names, as they came (A-468d).
|
||||
c := newRunFixture()
|
||||
p := c.start(t, 4, 9)
|
||||
comp := stepNamed(p, "", "composition")
|
||||
bs := stepSeatOf(t, "mesh-wants")
|
||||
raw := []byte(`{"id":"` + comp.Ask + `","pipeline":"main.4.app","step":"composition","module":"app","commit":"c1","fingerprint":"src2:aa","node":"n1","outcome":"passed",` +
|
||||
`"shared":[{"machine":"ace","run":"main.9.ace","ask":"r1"}]}`)
|
||||
ev, err := bs.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.h.OnStepEndedShared(bs, bs.Prefix+"n1", ev, bs.Shared(raw)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := stepNamed(c.pipeline(p), "", "composition"); len(s.Shared) != 1 || s.Shared[0] != (seat.Shared{Machine: "ace", Run: "main.9.ace", Ask: "r1"}) {
|
||||
t.Errorf("the composition step's shared = %+v, want mesh-wants' as it came", s.Shared)
|
||||
}
|
||||
}
|
||||
|
||||
// Asking a test run again moves every reader (design 47 G §727; A-469l, A-470d, A-470e): a rerun of one artifact's failed
|
||||
// step starts a new run under the same key, carrying the former readers, the one that did not pass marked; the new run's
|
||||
// passing end moves the other artifact's step to passed, marked a rerun by the caller and naming the new ask, and the
|
||||
// pipeline says pipeline-ended again; a step a person accepted does not move.
|
||||
func TestARerunOfATestRunMovesEveryReaderThatDidNotPass(t *testing.T) {
|
||||
// takeUp: the rerun's run is ended by the restart's take-up, its bounds gone while the runner was away; a carried reader
|
||||
// with rerun false, its step passed already, says no step-ended again (A-470l).
|
||||
for _, c := range []struct{ accepted, takeUp bool }{{false, false}, {true, false}, {false, true}} {
|
||||
accepted := c.accepted
|
||||
f := newRunFixture()
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
|
||||
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
|
||||
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
|
||||
p := f.start(t, 4, 9)
|
||||
e := f.store[p.Pipeline]
|
||||
e.p.Steps[0].Outcome = seat.StepResultOutcomePassed // the build
|
||||
f.store[p.Pipeline] = e
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
end := func(ask string, outcome string, rerun bool) {
|
||||
t.Helper()
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: outcome,
|
||||
Rerun: rerun, By: map[bool]string{true: "op@shanks"}[rerun], Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
end(unit.Ask, "failed", false)
|
||||
if q := f.pipeline(p); q.Outcome != seat.PipelineOutcomeFailed || q.EndCount != 1 || stepNamed(q, "app:tools", "unit").Outcome != "failed" {
|
||||
t.Fatalf("after the failing run the pipeline = %s (%d), tools %s", q.Outcome, q.EndCount, stepNamed(q, "app:tools", "unit").Outcome)
|
||||
}
|
||||
if accepted {
|
||||
e := f.store[p.Pipeline]
|
||||
i := stepIndex(e.p, "unit", "app:program")
|
||||
e.p.Steps[i].Outcome, e.p.Steps[i].Accepted = seat.StepResultOutcomeAccepted, &seat.Accepted{By: "op@shanks", Why: "known"}
|
||||
f.store[p.Pipeline] = e
|
||||
}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
|
||||
return Half{Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
|
||||
}, Acts: &actLog{}}
|
||||
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools", Why: "flaky"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := a.Result.Ask
|
||||
if c.takeUp {
|
||||
run := unit.Shared[0].Run
|
||||
r, rev, _ := f.runs.Get(run)
|
||||
r.Readers[0].Rerun, r.Readers[0].By = false, "" // the program's step passed already
|
||||
if _, err := f.runs.Update(run, r, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := f.store[p.Pipeline]
|
||||
e.p.Steps[stepIndex(e.p, "unit", "app:program")].Outcome = seat.StepResultOutcomePassed
|
||||
f.store[p.Pipeline] = e
|
||||
for k := range f.clock.bound {
|
||||
delete(f.clock.bound, k)
|
||||
}
|
||||
said := len(f.said)
|
||||
if n, err := f.h.TakeUp(); err != nil || n != 1 {
|
||||
t.Fatalf("take-up = %d, %v", n, err)
|
||||
}
|
||||
for _, ev := range f.said[said:] {
|
||||
if ev.Result.Artifact == "app:program" {
|
||||
t.Errorf("a carried reader with rerun false said %s again: %+v", ev.Result.Outcome, ev)
|
||||
}
|
||||
}
|
||||
q := f.pipeline(p)
|
||||
if stepNamed(q, "app:program", "unit").Outcome != seat.StepResultOutcomePassed || stepNamed(q, "app:tools", "unit").Outcome != seat.StepResultOutcomeCouldNotRun {
|
||||
t.Errorf("after the take-up program %s, tools %s; want passed kept, tools could not run",
|
||||
stepNamed(q, "app:program", "unit").Outcome, stepNamed(q, "app:tools", "unit").Outcome)
|
||||
}
|
||||
continue
|
||||
}
|
||||
r, _, _ := f.runs.Get(unit.Shared[0].Run)
|
||||
if again == "" || again == unit.Ask || r.Ask != again || r.Ended != nil || len(r.Readers) != 2 || r.Readers[0].Rerun == accepted || (!accepted && r.Readers[0].By != "op@shanks") {
|
||||
t.Fatalf("the rerun's run = %+v (ask %s), want a new run under the same key carrying the program, marked unless accepted", r, again)
|
||||
}
|
||||
end(again, "passed", true)
|
||||
q := f.pipeline(p)
|
||||
prog := stepNamed(q, "app:program", "unit")
|
||||
if accepted {
|
||||
if prog.Outcome != seat.StepResultOutcomeAccepted {
|
||||
t.Errorf("an accepted step moved to %s", prog.Outcome)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if prog.Outcome != seat.StepResultOutcomePassed || !prog.Rerun || prog.By != "op@shanks" || prog.Shared[0].Ask != again {
|
||||
t.Errorf("the program's step = %+v, want passed, a rerun by the caller, naming the new ask", prog)
|
||||
}
|
||||
if q.Outcome != seat.PipelineOutcomePassed || q.EndCount != 2 {
|
||||
t.Errorf("the pipeline = %s (%d), want passed at its second end", q.Outcome, q.EndCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The test runs' tools (design 47 G §728, §733 unit; A-466v, A-469g, A-469u, A-469v): test-runs answers each run with its
|
||||
// packages read from its output object, and a run whose object is gone without them, saying so; test-runs-drop records the
|
||||
// hand act first, ends the readers could not run naming the drop, and deletes the key and its object; with no hand act
|
||||
// recorded it does nothing.
|
||||
func TestTheTestRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit, lint := stepNamed(p, "app:program", "unit"), stepNamed(p, "app:program", "lint")
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "passed",
|
||||
Kept: []nodebuildgo.KeptResult{{Package: "app/x", Digest: "d1", Level: "unit"}}, Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pipes := memKept{}
|
||||
for k, e := range f.runs {
|
||||
put(t, pipes, k, e.r)
|
||||
}
|
||||
// The fill is said against the cap setting that refuses, never the bucket's 64 MiB ceiling (A-470g, A-470i).
|
||||
f.s.Runs.CapBytes, f.fill = 1<<20, 9
|
||||
bucket := f.s.RunsFill
|
||||
f.s.RunsFill = func() (seat.Fill, error) { fl, err := bucket(); fl.Cap = 64 << 20; return fl, err }
|
||||
read := RunsRead{Runs: pipes, Object: f.objects.Get, Fill: f.s.TestRunsFill}
|
||||
a, err := read.TestRuns(seat.TestRunsInput{Level: "unit", Module: "app"})
|
||||
if err != nil || len(a.Runs) != 1 || len(a.Runs[0].Packages) != 1 || a.Runs[0].Packages[0] != (seat.TestRunPackage{Package: "app/x", Digest: "d1"}) ||
|
||||
a.Runs[0].Key != unit.Shared[0].Run || len(a.Runs[0].Readers) != 2 || a.Fill.Cap != 1<<20 || a.Fill.Bytes != 9 {
|
||||
t.Fatalf("test-runs = %+v, %v; want the unit run with its package and readers, and the fill against the cap setting", a, err)
|
||||
}
|
||||
delete(f.objects, unit.Shared[0].Run) // gone, whatever step-outputs' age: the run answers without its packages
|
||||
if a, _ := read.TestRuns(seat.TestRunsInput{Level: "unit"}); len(a.Runs) != 1 || a.Runs[0].Gone != Gone || len(a.Runs[0].Packages) != 0 {
|
||||
t.Errorf("a run whose output is gone = %+v", a.Runs)
|
||||
}
|
||||
acts := &actLog{}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, Acts: acts}
|
||||
if _, err := (Verbs{Hearer: f.h, Starter: f.s}).TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"}); err == nil {
|
||||
t.Fatal("a drop with no hand act recorded was done")
|
||||
}
|
||||
f.objects[lint.Shared[0].Run] = []byte("{}")
|
||||
d, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: lint.Shared[0].Run, Why: "stuck"})
|
||||
if err != nil || len(*acts) != 1 || len(d.Ended) != 2 || d.Run.Key != lint.Shared[0].Run || len(f.drops) != 1 {
|
||||
t.Fatalf("drop = %+v, %v; acts %v, drops %v", d, err, *acts, f.drops)
|
||||
}
|
||||
if _, _, err := f.runs.Get(lint.Shared[0].Run); err != ErrNoRun || f.objects[lint.Shared[0].Run] != nil {
|
||||
t.Errorf("after the drop the key is %v and the object %q, want both gone", err, f.objects[lint.Shared[0].Run])
|
||||
}
|
||||
if s := stepNamed(f.pipeline(p), "app:tools", "lint"); s.Outcome != seat.StepResultOutcomeCouldNotRun || !strings.Contains(s.Why, "dropped by op@shanks") {
|
||||
t.Errorf("a reader of the dropped run = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// racingRuns is test-runs with a write coming between a caller's read and its next update or delete: each hook is called
|
||||
// once, just before that write.
|
||||
type racingRuns struct {
|
||||
testRunsMem
|
||||
beforeUpdate, beforeDelete func()
|
||||
}
|
||||
|
||||
func (r *racingRuns) Update(key string, v Run, rev uint64) (uint64, error) {
|
||||
if b := r.beforeUpdate; b != nil {
|
||||
r.beforeUpdate = nil
|
||||
b()
|
||||
}
|
||||
return r.testRunsMem.Update(key, v, rev)
|
||||
}
|
||||
|
||||
func (r *racingRuns) Delete(key string, rev uint64) error {
|
||||
if b := r.beforeDelete; b != nil {
|
||||
r.beforeDelete = nil
|
||||
b()
|
||||
}
|
||||
return r.testRunsMem.Delete(key, rev)
|
||||
}
|
||||
|
||||
// No write of test-runs or its bound keys puts back, ends or deletes a run from a read made before its latest change (design
|
||||
// 47 G §725, §726, §727, §728; the kind test 109 met): each write is a compare-and-set at the revision read, retried from the
|
||||
// latest read, and a judgement of a run ends only the run under the ask it judged.
|
||||
func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
|
||||
asked := func(f *runFixture, run string) {
|
||||
t.Helper()
|
||||
r, rev, _ := f.runs.Get(run)
|
||||
r.TestRun = seat.TestRun{Level: r.Level, Digest: r.Digest, Seat: r.Seat, Ask: "a2"}
|
||||
if _, err := f.runs.Update(run, r, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
race func(t *testing.T, f *runFixture, p seat.Pipeline, run string)
|
||||
}{
|
||||
{"an expiry does not end a run asked again while its ask was dropped", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
_ = f.h.RunTaken(unit.Ask, "ace")
|
||||
f.h.DropRun = func(string, string, string, string) error { asked(f, run); return nil }
|
||||
delete(f.clock.bound, BoundPrefix+run) // the bus's expiry deletes it
|
||||
_ = f.h.RunExpired(BoundPrefix + run)
|
||||
}},
|
||||
{"the last reader to leave ends the run in its own write, so a step joining then asks its own", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
f.h.DropRun = func(string, string, string, string) error {
|
||||
_, _, _ = f.s.claimRun(Asked{Run: run, ID: "a2", Level: "unit", Seat: "node-build-go",
|
||||
Readers: []seat.TestRunReader{{Pipeline: "main.5.app", Artifact: "app:program"}}})
|
||||
return nil
|
||||
}
|
||||
for _, a := range []string{"app:program", "app:tools"} {
|
||||
f.h.leaveRun(p.Pipeline, stepNamed(p, a, "unit"), "cancelled")
|
||||
}
|
||||
}},
|
||||
{"the age does not delete a run asked again since it was read", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
_ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil)
|
||||
runs := &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
|
||||
a := &Ager{Runs: runs, RunsKept: time.Hour, Pipelines: agedDeleter{f.store}, Aged: agedState{}, Now: func() time.Time { return agedNow },
|
||||
Log: func(from uint64, limit int) ([]LoggedEntry, error) {
|
||||
if from > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
return []LoggedEntry{{Sequence: 1, Entry: seat.LogEntry{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:tools",
|
||||
What: seat.LogEntryWhatEnded, At: agedNow.Add(-2 * time.Hour)}}}, nil
|
||||
}}
|
||||
a.Age()
|
||||
}},
|
||||
{"test-runs-drop does not delete a run asked again under its key", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeDelete: func() { asked(f, run) }}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, Acts: &actLog{}}
|
||||
if _, err := v.TestRunsDrop("op@shanks", seat.TestRunsDropInput{Key: run, Why: "stuck"}); err == nil {
|
||||
t.Error("a drop whose key holds a run asked since answers no error")
|
||||
}
|
||||
}},
|
||||
{"a step-progress heard as the run ends leaves no clock", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
_ = f.h.RunTaken(unit.Ask, "ace")
|
||||
f.clock.before = func() { _ = f.h.EndRun(run, seat.TestRun{Outcome: seat.StepResultOutcomePassed}, nil) }
|
||||
_ = f.h.RunProgress(unit.Ask)
|
||||
if _, ok := f.clock.bound[BoundPrefix+run]; ok {
|
||||
t.Error("a clock written from a read before the run's end is kept")
|
||||
}
|
||||
asked(f, run) // the key's run is asked again, so the check below holds
|
||||
}},
|
||||
{"a clock of a run not taken ends nothing", func(t *testing.T, f *runFixture, p seat.Pipeline, run string) {
|
||||
asked(f, run)
|
||||
_ = f.h.RunExpired(BoundPrefix + run)
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
run := stepNamed(p, "app:program", "unit").Shared[0].Run
|
||||
c.race(t, f, p, run)
|
||||
if r, _, err := f.runs.Get(run); err != nil || r.Ask != "a2" || r.Ended != nil {
|
||||
t.Errorf("the key's run = %+v, %v; want the run asked again under a2, running", r, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A late signal of a replaced run arms no bound and is said in the journal (A-470n).
|
||||
t.Run("a late ask-taken or step-progress of a replaced run moves nothing", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
asked(f, run)
|
||||
delete(f.clock.bound, UntakenPrefix+run)
|
||||
_ = f.h.RunTaken(unit.Ask, "ace")
|
||||
_ = f.h.RunProgress(unit.Ask)
|
||||
late := 0
|
||||
for _, l := range f.logged {
|
||||
if strings.Contains(l, unit.Ask) && strings.Contains(l, "moves nothing") {
|
||||
late++
|
||||
}
|
||||
}
|
||||
if _, ok := f.clock.bound[BoundPrefix+run]; ok || late != 2 {
|
||||
t.Errorf("bounds %v, journal %q; want no clock and both late signals journaled", f.clock.bound, f.logged)
|
||||
}
|
||||
})
|
||||
// A run's end is written as its step-ended is heard, before the finding the step says (A-470o).
|
||||
t.Run("the run ends before the step's finding is written", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
reg := catalog.Registration{Module: "app", Branches: []string{"main"}, Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "required",
|
||||
Integration: "bypass", Composition: "bypass", Review: "bypass", Approval: "bypass"}}}
|
||||
f.s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
endedFirst := false
|
||||
f.h.Failures = &Failures{Findings: newHeld(), Author: func(string, string) (string, error) {
|
||||
r, _, _ := f.runs.Get(run)
|
||||
endedFirst = r.Ended != nil
|
||||
return "alice", nil
|
||||
}}
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !endedFirst {
|
||||
t.Error("the step's finding was written before its run's end")
|
||||
}
|
||||
})
|
||||
// A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read,
|
||||
// carrying that reader, never answering the change as an error nor putting back the earlier readers.
|
||||
t.Run("a rerun writes its run from the latest read", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.h.TestRuns = &racingRuns{testRunsMem: f.runs, beforeUpdate: func() {
|
||||
r, rev, _ := f.runs.Get(run)
|
||||
r.Readers = append(r.Readers, RunReader{Pipeline: "main.9.app", Artifact: "app:program"})
|
||||
_, _ = f.runs.Update(run, r, rev)
|
||||
}}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
|
||||
return Half{Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
|
||||
}, Acts: &actLog{}}
|
||||
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"})
|
||||
if err != nil {
|
||||
t.Fatalf("a rerun whose read was changed before its write: %v", err)
|
||||
}
|
||||
r, _, _ := f.runs.Get(run)
|
||||
joined := false
|
||||
for _, o := range r.Readers {
|
||||
joined = joined || o.Pipeline == "main.9.app"
|
||||
}
|
||||
if r.Ask != a.Result.Ask || r.Ended != nil || !joined || len(r.Readers) != 3 {
|
||||
t.Errorf("the rerun's run = %+v; want the new ask running, carrying the reader that joined", r)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -27,8 +27,8 @@ type Verbs struct {
|
||||
Candidate func(group, branch string, revision int64) (Half, error)
|
||||
// Acts records a hand act before the verb does it (rerun, accept, cancel); a nil Acts refuses the verb, nothing recorded.
|
||||
Acts views.HandActs
|
||||
// Drop drops the bound of an ask a cancelled pipeline no longer waits on.
|
||||
Drop func(ask string) error
|
||||
// Runs answers the read verb test-runs, answered by the process that holds test-runs-cap-mib (A-470i).
|
||||
Runs RunsRead
|
||||
}
|
||||
|
||||
func (v Verbs) now() time.Time { return v.Hearer.now() }
|
||||
@@ -185,8 +185,8 @@ func (v Verbs) Accept(caller string, in seat.AcceptInput) (seat.AcceptAnswer, er
|
||||
return seat.AcceptAnswer{Result: p.Steps[i]}, nil
|
||||
}
|
||||
|
||||
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and the bound of each ask dropped, recorded in
|
||||
// the hand-act log first (§394); it says pipeline-ended.
|
||||
// Cancel ends a running pipeline `cancelled`, each step not ended cancelled and each step of a test run leaving it, recorded
|
||||
// in the hand-act log first (§394, §726); it says pipeline-ended.
|
||||
func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, error) {
|
||||
h := v.Hearer
|
||||
recorded := false
|
||||
@@ -205,14 +205,14 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
|
||||
recorded = true
|
||||
}
|
||||
now := h.now()
|
||||
var drop []string
|
||||
var leave []seat.StepResult
|
||||
for i := range p.Steps {
|
||||
s := &p.Steps[i]
|
||||
if s.Outcome != seat.StepResultOutcomeRunning && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
if !open(*s) && s.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
if s.Ask != "" {
|
||||
drop = append(drop, s.Ask)
|
||||
if sharedRun(*s) != "" && s.Seat != wants.Seat {
|
||||
leave = append(leave, *s) // it leaves its test run, dropped only with the last reader (§726)
|
||||
}
|
||||
s.Outcome, s.By, s.Why, s.Ended = seat.StepResultOutcomeCancelled, caller, in.Why, &now
|
||||
}
|
||||
@@ -223,12 +223,8 @@ func (v Verbs) Cancel(caller string, in seat.CancelInput) (seat.CancelAnswer, er
|
||||
} else if err != nil {
|
||||
return seat.CancelAnswer{}, fmt.Errorf("pipeline %s cannot be written: %w", in.Pipeline, err)
|
||||
}
|
||||
if v.Drop != nil {
|
||||
for _, ask := range drop {
|
||||
if err := v.Drop(ask); err != nil {
|
||||
h.logf("the bound of ask %s of the cancelled %s is not dropped: %v", ask, in.Pipeline, err)
|
||||
}
|
||||
}
|
||||
for _, s := range leave {
|
||||
h.leaveRun(in.Pipeline, s, "the pipeline "+in.Pipeline+" is cancelled: "+in.Why)
|
||||
}
|
||||
return seat.CancelAnswer{Pipeline: p}, h.said(p, false, true)
|
||||
}
|
||||
@@ -242,6 +238,11 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
if in.Step == seat.StepResultStepApproval {
|
||||
return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun")
|
||||
}
|
||||
if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil {
|
||||
if i, err := stepOf(p, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) {
|
||||
return v.rerunTest(caller, in, p.Steps[i])
|
||||
}
|
||||
}
|
||||
var ask *Asked
|
||||
p, _, err := v.change(in.Pipeline, seat.EventStepStarted, func() error {
|
||||
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
|
||||
@@ -289,7 +290,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449)
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, v.Starter.Arm, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -301,6 +302,173 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &r}, nil
|
||||
}
|
||||
|
||||
// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one
|
||||
// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set
|
||||
// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass.
|
||||
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) {
|
||||
h := v.Hearer
|
||||
run := sharedRun(st)
|
||||
if open(st) {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the %s step of %s is running: it is not asked again", in.Step, in.Pipeline)
|
||||
}
|
||||
r, rev, err := h.TestRuns.Get(run)
|
||||
if err != nil && !errors.Is(err, ErrNoRun) {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the test run %s is not read: %w", run, err)
|
||||
}
|
||||
exists := err == nil
|
||||
reader := seat.TestRunReader{Pipeline: in.Pipeline, Artifact: st.Artifact}
|
||||
read := exists && r.Ended != nil && r.Outcome != seat.StepResultOutcomeCouldNotRun && st.Ended != nil && r.Started != nil && r.Started.After(*st.Ended)
|
||||
join := exists && r.Ended == nil
|
||||
var ask *Asked
|
||||
if !join && !read {
|
||||
p, _, err := h.Store.Get(in.Pipeline)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
if ask, err = v.askOf(p, st, caller, h.now()); err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
}
|
||||
var asker *seat.SharedAskedBy
|
||||
if join || read {
|
||||
if asker, err = askerOf(h.Store, r.Ask); err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
if asker != nil && *asker == (seat.SharedAskedBy{Pipeline: reader.Pipeline, Artifact: reader.Artifact}) {
|
||||
asker = nil
|
||||
}
|
||||
}
|
||||
event := seat.EventStepStarted
|
||||
if read {
|
||||
event = seat.EventStepEnded
|
||||
}
|
||||
p, _, err := v.change(in.Pipeline, event, func() error {
|
||||
return v.record(seat.VerbRerun, fmt.Sprintf("asked the %s step of %s again", in.Step, in.Pipeline), in.Why)
|
||||
}, func(p *seat.Pipeline, now time.Time) (int, error) {
|
||||
switch p.Outcome {
|
||||
case seat.PipelineOutcomeCancelled, seat.PipelineOutcomeSuperseded:
|
||||
return -1, fmt.Errorf("the pipeline %s is %s: a step of it is not asked again", p.Pipeline, p.Outcome)
|
||||
}
|
||||
i, err := stepOf(*p, in.Step, in.Artifact)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
s := &p.Steps[i]
|
||||
*s = seat.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
|
||||
Outcome: seat.StepResultOutcomeRunning, By: caller, Rerun: true, Started: &now}
|
||||
switch {
|
||||
case ask != nil:
|
||||
s.Ask, s.Shared = ask.ID, []seat.Shared{{Run: run, Ask: ask.ID}}
|
||||
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode // until ask-taken (A-469b)
|
||||
case read:
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
|
||||
s.Outcome, s.Why, s.Node, s.Ended = r.Outcome, r.Said, r.Node, &now
|
||||
if r.Tests != nil {
|
||||
t := *r.Tests
|
||||
s.Tests = &t
|
||||
}
|
||||
default:
|
||||
s.Shared = []seat.Shared{{Run: run, Ask: r.Ask, AskedBy: asker}}
|
||||
if r.Node == "" {
|
||||
s.Outcome, s.On = seat.StepResultOutcomeWaitingForABuildNode, seat.WaitOnBuildNode
|
||||
}
|
||||
}
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
|
||||
}
|
||||
for j := range p.Steps {
|
||||
if p.Steps[j].Step == seat.StepResultStepApproval && p.Steps[j].Outcome == seat.StepResultOutcomeNotAsked {
|
||||
p.Steps[j].Outcome, p.Steps[j].Why = seat.StepResultOutcomeWaiting, ""
|
||||
}
|
||||
}
|
||||
return i, nil
|
||||
})
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
h.pin(p)
|
||||
switch {
|
||||
case join || read:
|
||||
v.Starter.joinRun(run, reader)
|
||||
default:
|
||||
now := h.now()
|
||||
a := *ask
|
||||
a.Run = run
|
||||
// The new run is written from the latest read of the key: a write between (a reader leaving, a join) is read again and
|
||||
// its readers carried, never put back; a run another step started since is said, not replaced.
|
||||
for {
|
||||
value := Run{TestRun: seat.TestRun{Level: st.Step, Digest: r.Digest, Seat: st.Seat, Ask: ask.ID, Started: &now}}
|
||||
// Every former reader is carried, each that did not pass marked to move with the new run by this caller (A-470d).
|
||||
for _, o := range r.Readers {
|
||||
if o.Of() == reader {
|
||||
continue
|
||||
}
|
||||
o.Rerun, o.By = !v.passedReader(o, run), ""
|
||||
if o.Rerun {
|
||||
o.By = caller
|
||||
}
|
||||
value.Readers = append(value.Readers, o)
|
||||
}
|
||||
addReaders(&value, []seat.TestRunReader{reader})
|
||||
err := v.Starter.started(a, func() error {
|
||||
if !exists {
|
||||
return h.TestRuns.Create(run, value)
|
||||
}
|
||||
_, err := h.TestRuns.Update(run, value, rev)
|
||||
return err
|
||||
})
|
||||
if errors.Is(err, ErrChanged) || errors.Is(err, ErrExists) {
|
||||
got, gotRev, gerr := h.TestRuns.Get(run)
|
||||
switch {
|
||||
case errors.Is(gerr, ErrNoRun):
|
||||
r, rev, exists = Run{TestRun: seat.TestRun{Digest: r.Digest}}, 0, false
|
||||
continue
|
||||
case gerr != nil:
|
||||
err = gerr
|
||||
case got.Ended == nil:
|
||||
err = fmt.Errorf("the run of ask %s was asked under the key since it was read", got.Ask)
|
||||
default:
|
||||
r, rev, exists = got, gotRev, true
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but its test run %s is not: %w", run, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
i, err := stepOf(p, in.Step, in.Artifact)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
res := p.Steps[i]
|
||||
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil
|
||||
}
|
||||
|
||||
// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run.
|
||||
func (v Verbs) passedReader(reader RunReader, run string) bool {
|
||||
p, _, err := v.Hearer.Store.Get(reader.Pipeline)
|
||||
if err != nil {
|
||||
return true // a pipeline gone has no step to move
|
||||
}
|
||||
for _, s := range p.Steps {
|
||||
if s.Artifact == reader.Artifact && sharedRun(s) == run {
|
||||
switch s.Outcome {
|
||||
case seat.StepResultOutcomePassed, seat.StepResultOutcomeCarried, seat.StepResultOutcomeAccepted,
|
||||
seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeCancelled:
|
||||
return true
|
||||
}
|
||||
return s.Accepted != nil
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice
|
||||
// under one id is kept once), saying it is a rerun and by whom.
|
||||
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
|
||||
@@ -325,7 +493,7 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
|
||||
} else {
|
||||
return nil, fmt.Errorf("the catalog half of %s is not held yet: the step is asked when it is", p.Branch)
|
||||
}
|
||||
_, placed := Place(p, reg, half.Entries, now, v.Starter.Name)
|
||||
_, placed := place(p, reg, half.Entries, now, v.Starter.Name, false)
|
||||
base := AskID(startOf(p, now), p.Pipeline, s.Artifact, s.Step)
|
||||
for _, a := range placed {
|
||||
if a.ID != base {
|
||||
|
||||
@@ -35,8 +35,9 @@ func verbsFixture() (Verbs, memStore, *[]string, *stepAsks, *actLog, *[]string)
|
||||
asks, acts, dropped := &stepAsks{}, &actLog{}, &[]string{}
|
||||
s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true },
|
||||
Now: h.Now}
|
||||
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts,
|
||||
Drop: func(ask string) error { *dropped = append(*dropped, ask); return nil }}, store, said, asks, acts, dropped
|
||||
h.DropRun = func(_, _, ask, _ string) error { *dropped = append(*dropped, ask); return nil }
|
||||
return Verbs{Hearer: h, Starter: s, View: func(string) (Half, bool) { return Half{Entries: entries}, true }, Acts: acts},
|
||||
store, said, asks, acts, dropped
|
||||
}
|
||||
|
||||
func stepNamed(p seat.Pipeline, artifact, step string) seat.StepResult {
|
||||
@@ -81,7 +82,8 @@ func TestReviewEndsThePlacedReviewStepWithItsProvenCaller(t *testing.T) {
|
||||
|
||||
// accept takes only a failed optional step, writes its hand act first, and refuses without one; rerun asks the step again
|
||||
// under a new ask saying it is a rerun and its caller, reopens a failed pipeline and asks nothing of a refused one; cancel ends
|
||||
// a running pipeline cancelled, each step cancelled and each ask's bound dropped (design 47 G §22, §30, §325, §394).
|
||||
// a running pipeline cancelled, each step cancelled, no ask's bound left to drop (design 47 G §22, §30, §325, §394; §647 as
|
||||
// A-466y amends it).
|
||||
func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testing.T) {
|
||||
v, store, said, asks, acts, dropped := verbsFixture()
|
||||
end := func(step, outcome string) {
|
||||
@@ -134,7 +136,7 @@ func TestAcceptRerunAndCancelWriteByCompareAndSetAndRecordTheirHandAct(t *testin
|
||||
}
|
||||
// A cancel ends the pipeline cancelled and drops the bounds of the asks still held.
|
||||
c, err := v.Cancel("operator", seat.CancelInput{Pipeline: pkey, Why: "wrong branch"})
|
||||
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) == 0 {
|
||||
if err != nil || c.Pipeline.Outcome != "cancelled" || c.Pipeline.EndCount != 1 || len(*dropped) != 0 {
|
||||
t.Fatalf("cancel = %+v, %v, dropped %v", c.Pipeline, err, *dropped)
|
||||
}
|
||||
if s := stepNamed(c.Pipeline, "app:program", "lint"); s.Outcome != "cancelled" || s.By != "operator" {
|
||||
|
||||
@@ -55,9 +55,12 @@ func TestManifest(t *testing.T) {
|
||||
{"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat &&
|
||||
equalSets(m.Claims[0].Serves, seat.Verbs())},
|
||||
// §27, §85, §330, §355, §77: the old state sections.
|
||||
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "toolchains", "findings", "first-views", "consumers"})},
|
||||
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})},
|
||||
// §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live.
|
||||
{"bounds is key-ttl", keyTTL["bounds"]},
|
||||
// §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling.
|
||||
{"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64},
|
||||
{"the test runs' settings", runSettingsOf(raw)},
|
||||
// RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB.
|
||||
{"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256},
|
||||
{"kept 128 MiB", stateMiB[seat.StateKept] == 128},
|
||||
@@ -78,10 +81,13 @@ func TestManifest(t *testing.T) {
|
||||
{"consumes", equalSets(m.Consumes, []string{
|
||||
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
|
||||
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken",
|
||||
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
|
||||
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
|
||||
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"})},
|
||||
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed",
|
||||
// §647 as A-466z amends it: no holder is read from the seat-holders view.
|
||||
"seat:mesh-module-assigner.views.seat-holders"})},
|
||||
// RN2 (§21, §195, §200, §77): the registrations and the catalog half asked by their verbs, the first views read and
|
||||
// asked again by hand (a hand act).
|
||||
{"invokes", equalSets(m.Invokes, []string{"seat:mesh-catalog.registrations",
|
||||
@@ -107,7 +113,8 @@ func TestManifest(t *testing.T) {
|
||||
"seat:node-build-node.accept.delivery", "seat:node-build-image.accept.check", "seat:node-build-image.accept.delivery"})},
|
||||
{"uses operator-channel", equalSets(m.Uses, []string{"operator-channel"})},
|
||||
// RN25 (§539): the runner's tools carry the prefix runner_.
|
||||
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append"})},
|
||||
{"first-view tools", equalSets(m.Tools, []string{"runner_first_views", "runner_first_view_again", "runner_health", "runner_unlogged", "runner_unlogged_append",
|
||||
"runner_test_runs", "runner_test_runs_drop"})},
|
||||
// A-305 (ADR 0227 rule 8, ADR 0240 rule 2): the process is judged by its unit, as every process of mesh-delivery,
|
||||
// never by a tool of its own; runner_health stays a tool.
|
||||
{"health is the unit", healthTool(raw) == "unit"},
|
||||
@@ -175,3 +182,43 @@ func equalSets(a, b []string) bool {
|
||||
slices.Sort(y)
|
||||
return slices.Equal(x, y)
|
||||
}
|
||||
|
||||
// runSettingsOf says whether the manifest holds the test runs' four settings (design 47 G §725; A-469w, A-469x): the cap a
|
||||
// whole number of at most 64, the three times Go durations, each written into the file its environment names.
|
||||
func runSettingsOf(raw []byte) bool {
|
||||
var m struct {
|
||||
Config map[string]map[string]any `json:"config"`
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
Content string `json:"content"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return false
|
||||
}
|
||||
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string"}
|
||||
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) {
|
||||
return false
|
||||
}
|
||||
env := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
for _, v := range r.Env {
|
||||
env[v] = true
|
||||
}
|
||||
}
|
||||
for key, kind := range want {
|
||||
if m.Config[key]["type"] != kind {
|
||||
return false
|
||||
}
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
found = found || (r.Type == "file" && r.Content == "${config:"+key+"}" && env[r.Path])
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -18,7 +18,9 @@
|
||||
"rerun",
|
||||
"accept",
|
||||
"cancel",
|
||||
"ask-again"
|
||||
"ask-again",
|
||||
"test-runs",
|
||||
"test-runs-drop"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -27,7 +29,9 @@
|
||||
"runner_first_view_again",
|
||||
"runner_health",
|
||||
"runner_unlogged",
|
||||
"runner_unlogged_append"
|
||||
"runner_unlogged_append",
|
||||
"runner_test_runs",
|
||||
"runner_test_runs_drop"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-delivery-indexer.take-in.taken",
|
||||
@@ -41,13 +45,17 @@
|
||||
"node-build-go.ask-taken",
|
||||
"node-build-node.ask-taken",
|
||||
"node-build-image.ask-taken",
|
||||
"node-build-go.step-progress",
|
||||
"node-build-node.step-progress",
|
||||
"node-build-image.step-progress",
|
||||
"mesh-wants.step-ended",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-conditions.findings-asked",
|
||||
"mesh-delivery.recovery-merged",
|
||||
"mesh-catalog.seat-definition-changed",
|
||||
"mesh-catalog.seat-definition-removed"
|
||||
"mesh-catalog.seat-definition-removed",
|
||||
"seat:mesh-module-assigner.views.seat-holders"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-catalog.seat-definitions",
|
||||
@@ -79,6 +87,34 @@
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"config": {
|
||||
"test-runs-cap-mib": {
|
||||
"type": "number",
|
||||
"description": "MiB of the state test-runs at which the runner asks no new test run and ends the step could not run, naming the fill (design 47 G section 725, A-469d); the manifest's max-mib is the ceiling.",
|
||||
"minimum": 1,
|
||||
"maximum": 64,
|
||||
"trusted": false,
|
||||
"default": 16
|
||||
},
|
||||
"test-runs-kept": {
|
||||
"type": "string",
|
||||
"description": "how long an ended test run and its output object are kept after its end, a Go duration such as 168h (design 47 G section 725, A-469x, A-470a); at most 168h, so the pipeline naming its key is still kept.",
|
||||
"trusted": false,
|
||||
"default": "168h"
|
||||
},
|
||||
"test-step-bound": {
|
||||
"type": "string",
|
||||
"description": "how long a taken test run's ask may say no progress before the run ends could not run, a Go duration such as 30m, counted from the build seat's ask-taken and again from each step-progress (design 47 G section 725, A-466n, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "30m"
|
||||
},
|
||||
"run-untaken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "24h"
|
||||
}
|
||||
},
|
||||
"state": [
|
||||
{
|
||||
"name": "pipelines",
|
||||
@@ -107,6 +143,12 @@
|
||||
"history": 1,
|
||||
"key-ttl": true
|
||||
},
|
||||
{
|
||||
"name": "test-runs",
|
||||
"history": 1,
|
||||
"max-mib": 64,
|
||||
"key-ttl": true
|
||||
},
|
||||
{
|
||||
"name": "findings",
|
||||
"history": 1
|
||||
@@ -162,6 +204,38 @@
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "test-runs-cap",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/test-runs-cap-mib",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:test-runs-cap-mib}"
|
||||
},
|
||||
{
|
||||
"id": "test-runs-kept",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/test-runs-kept",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:test-runs-kept}"
|
||||
},
|
||||
{
|
||||
"id": "test-step-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/test-step-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:test-step-bound}"
|
||||
},
|
||||
{
|
||||
"id": "run-untaken-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/run-untaken-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:run-untaken-bound}"
|
||||
},
|
||||
{
|
||||
"id": "pipeline",
|
||||
"type": "process",
|
||||
@@ -171,7 +245,11 @@
|
||||
"./mesh-delivery-pipeline"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_RUNNER_TEST_RUNS_CAP_FILE": "${dir:state}/test-runs-cap-mib",
|
||||
"MESH_RUNNER_TEST_RUNS_KEPT_FILE": "${dir:state}/test-runs-kept",
|
||||
"MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound",
|
||||
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound"
|
||||
},
|
||||
"user": "mesh-delivery-pipeline",
|
||||
"health": {
|
||||
|
||||
@@ -67,6 +67,11 @@ type Request struct {
|
||||
// them (the definitions of the seat mesh-catalog, resolved), raw so that this package names no seat (ADR 0635 §4, ADR 0636
|
||||
// §1). The judge and the composer read them. None is no list at hand: a composer handed none judges no machine (ADR 0636 §1).
|
||||
Seats []json.RawMessage `json:"seats,omitempty"`
|
||||
// BlameTries bounds the composer's search for the modules a machine that does not compose blames: it composes at
|
||||
// most this many times more, the assigner's setting blame-tries, since the composer has no settings of its own
|
||||
// (design 47 G §721, §729; A-469i). None is no search: the composer blames only what its composition holds or
|
||||
// refuses (A-468i).
|
||||
BlameTries int `json:"blame_tries,omitempty"`
|
||||
}
|
||||
|
||||
// Case is one catalogue composed: every module's manifest by name, and per node the manifests kept at the build it runs.
|
||||
@@ -85,6 +90,11 @@ type Case struct {
|
||||
Verdicts map[string]json.RawMessage `json:"verdicts,omitempty"`
|
||||
Kept map[string]map[string]Manifest `json:"kept,omitempty"`
|
||||
Assignments map[string]json.RawMessage `json:"assignments,omitempty"`
|
||||
// None is, by module, the words of the revision's none for each module of the case the revision gives no fingerprint
|
||||
// (its build can never be made: builders disagree, no build module), the same word as the revision's. The assigner
|
||||
// fills it from the revision it reads for the run; the composer reads it alone to tell a build that can never be made
|
||||
// (design 47 G A-470j, A-470k).
|
||||
None map[string]string `json:"none,omitempty"`
|
||||
}
|
||||
|
||||
// Manifest is a module's manifest as its repository holds it (Raw, the bytes), or the digest the delivery indexer keeps
|
||||
@@ -168,6 +178,11 @@ type ManifestRead struct {
|
||||
// in words, why the composer did not judge the machine (an older node-engine's input that does not say how it composes,
|
||||
// novox/hq ADR 0593 §9): the machine is then neither composed nor refused, and a reader decides by this field alone,
|
||||
// never by the words of Reason.
|
||||
//
|
||||
// Blamed names each module the candidate's composition of the machine holds or refuses and the composition without the
|
||||
// candidate does not, with its problem (design 47 G §721). A composer of this shape always writes it, an empty list when
|
||||
// it blames none (A-469h); nil is a verdict with no such field, a composer from before §721, which its reader judges as
|
||||
// before it ("the composer names no culprits").
|
||||
type Machine struct {
|
||||
Composes bool `json:"composes"`
|
||||
NotJudged string `json:"not_judged,omitempty"`
|
||||
@@ -175,6 +190,10 @@ type Machine struct {
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
Refusals []string `json:"refusals,omitempty"`
|
||||
Declared []Path `json:"declared,omitempty"`
|
||||
Blamed []struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
} `json:"blamed,omitzero"`
|
||||
}
|
||||
|
||||
// Path is one path a declaration puts on the machine, with its kind (the resource's type), owner, group and mode as
|
||||
|
||||
@@ -57,6 +57,51 @@ func TestAManifestIsItsBytesOrItsDigest(t *testing.T) {
|
||||
t.Errorf("the known packages %#v went as %s and read back %#v (%v)", known, wire, back.Packages, err)
|
||||
}
|
||||
}
|
||||
// K1/AC4 (design 47 G §721, §729; A-469h, A-469i): a request's blame_tries and a verdict's blamed read back whole; a
|
||||
// verdict with no blamed (a composer from before §721) reads back as no field, told apart from an empty list.
|
||||
wire, err = json.Marshal(Request{V: Version, BlameTries: 20})
|
||||
if back, derr := Decode(wire); err != nil || derr != nil || back.BlameTries != 20 || !strings.Contains(string(wire), `"blame_tries":20`) {
|
||||
t.Errorf("blame_tries 20 went as %s and read back %d (%v, %v)", wire, back.BlameTries, err, derr)
|
||||
}
|
||||
// H3 (design 47 G A-470j, A-470k): a case's none, each module the revision gives no fingerprint with its none words,
|
||||
// reads back whole; a case naming none leaves the field off the wire.
|
||||
wire, err = json.Marshal(Request{V: Version, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{},
|
||||
None: map[string]string{"lidarr": "builders disagree: node"}}, {Name: "trunk", Manifests: map[string]Manifest{}}}})
|
||||
if back, derr := Decode(wire); err != nil || derr != nil || back.Cases[0].None["lidarr"] != "builders disagree: node" ||
|
||||
back.Cases[1].None != nil || !strings.Contains(string(wire), `"none":{"lidarr":"builders disagree: node"}`) ||
|
||||
strings.Count(string(wire), `"none"`) != 1 {
|
||||
t.Errorf("a case's none went as %s and read back %+v (%v, %v)", wire, back.Cases, err, derr)
|
||||
}
|
||||
one := Machine{}
|
||||
one.Blamed = append(one.Blamed, struct {
|
||||
Module string `json:"module"`
|
||||
Problem string `json:"problem"`
|
||||
}{Module: "a", Problem: "holds /etc/x"})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
wire string
|
||||
blamed int
|
||||
old bool
|
||||
}{
|
||||
{"blames one", `{"composes":false,"blamed":[{"module":"a","problem":"holds /etc/x"}]}`, 1, false},
|
||||
{"blames none", `{"composes":false,"blamed":[]}`, 0, false},
|
||||
{"an old composer", `{"composes":false}`, 0, true},
|
||||
} {
|
||||
var m Machine
|
||||
if err := json.Unmarshal([]byte(c.wire), &m); err != nil {
|
||||
t.Fatalf("%s: %v", c.name, err)
|
||||
}
|
||||
if len(m.Blamed) != c.blamed || (m.Blamed == nil) != c.old {
|
||||
t.Errorf("%s: %s read back blamed %#v", c.name, c.wire, m.Blamed)
|
||||
}
|
||||
again, err := json.Marshal(m)
|
||||
if err != nil || string(again) != c.wire {
|
||||
t.Errorf("%s: %s went again as %s (%v)", c.name, c.wire, again, err)
|
||||
}
|
||||
}
|
||||
if wire, _ := json.Marshal(one); string(wire) != `{"composes":false,"blamed":[{"module":"a","problem":"holds /etc/x"}]}` {
|
||||
t.Errorf("a machine blaming a went as %s", wire)
|
||||
}
|
||||
bad := Request{V: Version, Cases: []Case{{Name: "c", Manifests: map[string]Manifest{"x": {Digest: "nope"}}}}}
|
||||
if err := Fill(&bad, func(Manifest) ([]byte, error) { return nil, errors.New("not there") }); err == nil || !strings.Contains(err.Error(), "nope") {
|
||||
t.Fatalf("a digest the indexer does not hold: %v", err)
|
||||
|
||||
@@ -190,17 +190,23 @@ func brokerSubject(subject string) bool {
|
||||
return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.")
|
||||
}
|
||||
|
||||
// pipelineSubject hands the gate's `source-moved` and the runner's `pipeline-started` to the watchman (watch/pipelines.go,
|
||||
// design 47 G §12): the head of a watched branch and its pipeline. It says whether the subject was one of them.
|
||||
// pipelineSubject hands the gate's `source-moved`, the runner's `pipeline-started`, `step-started` and `step-ended` and the assigner's
|
||||
// `step-progress` to the watchman (watch/pipelines.go, watch/steps.go, design 47 G §12, §732): the head of a watched branch, its
|
||||
// pipeline and each step's clock. It says whether the subject was one of them.
|
||||
func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
switch {
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."):
|
||||
w.SourceMoved(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."):
|
||||
w.PipelineStarted(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."),
|
||||
strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."):
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."):
|
||||
w.StepEvent(body)
|
||||
w.StepStarted(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."):
|
||||
w.StepEvent(body)
|
||||
w.StepEnded(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshwants.Seat+".event."+meshwants.EventStepProgress+"."):
|
||||
w.WantsStepProgress(body)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
@@ -331,6 +332,55 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
|
||||
return finding(watch.FindingKey(watch.ScopeDelivery, stuckOf(askStream, ask1), watch.KindStuck))
|
||||
})
|
||||
until("node n1's silence", func() bool { return finding("machine.n1.silent") })
|
||||
|
||||
// Design 47 G §732 (test 108's watchman case): a composition step whose clock passed five minutes with no progress is
|
||||
// `pipeline-stands`; the assigner's `step-progress` for its ask clears it, and the step's own `step-ended` ends it.
|
||||
const pipeline, stepAsk = "main.9.web", "wants-ask-1"
|
||||
stepKey := pipeline + "/composition/"
|
||||
stepFinding := watch.FindingKey(watch.ScopeDelivery, watch.StepFindingID(stepKey), watch.KindPipelineStands)
|
||||
sayStep := func(event string, at time.Time) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "web", Commit: "c1", At: &at,
|
||||
Result: meshpipelinerunner.StepResult{Step: "composition", Ask: stepAsk, Outcome: meshpipelinerunner.StepResultOutcomeRunning}})
|
||||
if _, err := js.Publish(ctx, "mesh.seat.mesh-pipeline-runner.event."+event+".web.main.c1", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sayStep("step-started", time.Now().UTC().Add(-6*time.Minute))
|
||||
until("the composition step with no progress for six minutes", func() bool { return finding(stepFinding) })
|
||||
progress, _ := json.Marshal(meshwants.StepProgressEvent{Ask: stepAsk, Asker: "runner", At: time.Now().UTC(), Phase: "run"})
|
||||
if _, err := js.Publish(ctx, "mesh.seat.mesh-wants.event.step-progress.n1", progress); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
until("the step's progress clears it", func() bool { return !finding(stepFinding) })
|
||||
sayStep("step-started", time.Now().UTC().Add(-6*time.Minute))
|
||||
until("the step stands again", func() bool { return finding(stepFinding) })
|
||||
sayStep("step-ended", time.Now().UTC())
|
||||
until("the step's end clears it", func() bool { return !finding(stepFinding) })
|
||||
|
||||
// A step the runner's stands named at the start (seeded through the seat verb) and that no progress follows is
|
||||
// `pipeline-stands`; the approval waiting on a person is owed none.
|
||||
seeded := func(step string) string {
|
||||
return watch.FindingKey(watch.ScopeDelivery, watch.StepFindingID(seededPipeline+"/"+step+"/"), watch.KindPipelineStands)
|
||||
}
|
||||
until("the seeded composition step with no progress", func() bool { return finding(seeded("composition")) })
|
||||
if finding(seeded("approval")) {
|
||||
t.Error("a step waiting on a person raised pipeline-stands")
|
||||
}
|
||||
// A test step whose run's ask ended with no step-ended of its own stands five minutes (here the short bound) after that end.
|
||||
const runAsk = "run-ask-2"
|
||||
old := time.Now().UTC().Add(-time.Minute) // started longer ago than the short bound: it stands at once
|
||||
testStep, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: "main.6.web", Branch: "main", Module: "web", Commit: "c2", At: &old,
|
||||
Result: meshpipelinerunner.StepResult{Step: "unit", Artifact: "web:program", Ask: "t-ask", Outcome: meshpipelinerunner.StepResultOutcomeRunning,
|
||||
Shared: []meshpipelinerunner.Shared{{Run: "r2", Ask: runAsk}}}})
|
||||
if _, err := js.Publish(ctx, "mesh.seat.mesh-pipeline-runner.event.step-started.web.main.c2", testStep); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
testFinding := watch.FindingKey(watch.ScopeDelivery, watch.StepFindingID("main.6.web/unit/web:program"), watch.KindPipelineStands)
|
||||
until("the test step standing since its old start", func() bool { return finding(testFinding) })
|
||||
sayAsk(t, ctx, js, "step-ended", runAsk)
|
||||
until("the end of the run's ask begins the clock again, so the finding is cleared", func() bool { return !finding(testFinding) })
|
||||
until("the test step whose run ended with no end of its own, one bound after that end", func() bool { return finding(testFinding) })
|
||||
// A-274: the finding is said under the watchman's binding finding-events.
|
||||
until("a finding-written under the binding finding-events", func() bool {
|
||||
st, err := js.Stream(ctx, "WATCHMAN_FINDINGS")
|
||||
@@ -605,6 +655,9 @@ const (
|
||||
testWantsBucket = testAssigner + "_" + meshwants.StateWants
|
||||
)
|
||||
|
||||
// seededPipeline is the pipeline the stand-in runner's stands names.
|
||||
const seededPipeline = "main.5.web"
|
||||
|
||||
// serveCatalogHalf answers the catalog half's first views (design 47 G §614): the gate's one source branch, the stand-in
|
||||
// registered on it, and its trunk revision whose entry claims mesh-module-assigner.
|
||||
func serveCatalogHalf(t *testing.T, nc *nats.Conn) {
|
||||
@@ -617,7 +670,13 @@ func serveCatalogHalf(t *testing.T, nc *nats.Conn) {
|
||||
Entries: []meshcatalog.CatalogEntry{{Module: testAssigner, ManifestDigest: "d1", Manifest: map[string]any{"module": testAssigner,
|
||||
"claims": []any{map[string]any{"name": "mesh-module-assigner", "scope": "mesh"}}}}}}
|
||||
registrations := meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{{Module: testAssigner, Version: 1, Branches: []string{"release"}}}}
|
||||
for subject, handle := range map[string]nats.MsgHandler{"mesh.seat.mesh-catalog.call.revision": respond(revision),
|
||||
// The runner's stands (design 47 G §731, A-469p), which the watchman seeds its open steps from at its start: a composition step
|
||||
// waiting on the assigner, a test step reading a run, and an approval waiting on a person.
|
||||
stands := meshpipelinerunner.StandsAnswer{Stands: []meshpipelinerunner.Wait{
|
||||
{Pipeline: seededPipeline, Step: "composition", Ask: "seeded-ask", Module: "web", Branch: "main", Since: time.Now().UTC(), V: 1},
|
||||
{Pipeline: seededPipeline, Step: "approval", On: meshpipelinerunner.WaitOnPerson, Module: "web", Branch: "main", Since: time.Now().UTC(), V: 1}}}
|
||||
for subject, handle := range map[string]nats.MsgHandler{"mesh.seat.mesh-pipeline-runner.call.stands": respond(stands),
|
||||
"mesh.seat.mesh-catalog.call.revision": respond(revision),
|
||||
"mesh.seat.mesh-catalog.call.registrations": respond(registrations), "mesh.seat.mesh-delivery.call.source": respond(map[string]any{"branches": []string{"release"}})} {
|
||||
for _, s := range []string{subject, subject + ".>"} {
|
||||
sub, err := nc.Subscribe(s, handle)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"io"
|
||||
"log"
|
||||
@@ -400,6 +401,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
atCatalog := atHolder(calls.Call, holders, meshcatalog.Seat) // the indexer holds the seat mesh-catalog
|
||||
seedTakeIn(w, atCatalog, logger.Printf)
|
||||
seedVerdicts(w, atCatalog, logger.Printf)
|
||||
seedSteps(w, calls.Call, logger.Printf)
|
||||
calls.Close()
|
||||
}
|
||||
// The take-in events are the indexer's ordinary events (its emits), heard on the shared events stream through this
|
||||
@@ -507,6 +509,24 @@ func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMess
|
||||
logf("take-in: %s cannot be asked (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err)
|
||||
}
|
||||
|
||||
// standsVerb is the runner's seat verb that answers every step started and not ended (design 47 G §731, A-469p).
|
||||
const standsVerb = "seat:" + meshpipelinerunner.Seat + "." + meshpipelinerunner.VerbStands
|
||||
|
||||
// seedSteps asks the runner which steps stand now and keeps each, its clock begun at this start (Watchman.SeedSteps, design 47 G
|
||||
// §731, §732). A runner that does not answer is said: the steps are then known from the events heard after.
|
||||
func seedSteps(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) {
|
||||
raw, err := call(standsVerb, map[string]any{})
|
||||
if err == nil {
|
||||
var a meshpipelinerunner.StandsAnswer
|
||||
if err = json.Unmarshal(raw, &a); err == nil {
|
||||
w.SeedSteps(a.Stands)
|
||||
logf("steps: %d open steps the runner holds are watched from this start (%s)", len(a.Stands), standsVerb)
|
||||
return
|
||||
}
|
||||
}
|
||||
logf("steps: %s cannot be asked (%v), so a step open before this start is timed only once an event of it is heard", standsVerb, err)
|
||||
}
|
||||
|
||||
// seedVerdicts asks the indexer's verdicts and raises the findings they hold (design 47 G §500): a verdict said while the watchman was
|
||||
// down is then not lost. An indexer that does not answer is said: the findings are then raised from the verdicts heard after.
|
||||
func seedVerdicts(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) {
|
||||
|
||||
@@ -55,6 +55,9 @@ func (w *Watchman) StepProgress(seat, node string, body []byte) {
|
||||
w.say("a step-progress of %s does not read, left: %v", seat, err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.stepsHeardOf(e.Ask, e.At)
|
||||
w.mu.Unlock()
|
||||
w.askClock(seat, node, e.Ask, e.At)
|
||||
}
|
||||
|
||||
@@ -75,6 +78,7 @@ func (w *Watchman) AskEnded(seat string, body []byte) {
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.stepsHeardOf(e.Ask, e.At)
|
||||
w.moved(bucket, e.Ask)
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ func (w *Watchman) timeHead(branch string) {
|
||||
Headline: "A change has started no pipeline",
|
||||
Explanation: "The head of the watched branch " + branch + " (source revision " + fmt.Sprint(h.revision) + ") moved " +
|
||||
HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.",
|
||||
Needs: "the runner to start the pipeline (its pipeline_stands and list tools say where it is)"}
|
||||
Needs: "the runner to start the pipeline (its verbs mesh-pipeline-runner.stands and mesh-pipeline-runner.list say where it is)"}
|
||||
left := HeadBound - w.now().Sub(h.at)
|
||||
if left > 0 {
|
||||
w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f})
|
||||
@@ -132,7 +132,7 @@ func (w *Watchman) StepEvent(body []byte) {
|
||||
w.say("a step event does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
id := e.Pipeline + "/" + e.Result.Step
|
||||
id := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact) // design 47 G A-469o
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for seat, q := range w.queued {
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// The watchman times every step that reads a run (design 47 G §731, §732, A-468j, A-469, A-469o, A-469p): each step started and
|
||||
// not ended is kept in memory under `<pipeline>/<step>/<artifact>` (the artifact empty for a module's step), its clock begun at its
|
||||
// `step-started` and begun again at each `mesh-wants` `step-progress` whose ask is the step's own ask or one its `shared` names, at
|
||||
// each build seat `step-progress` of such an ask, and at the `step-ended` of such an ask. A clock past the five-minute progress
|
||||
// bound raises `pipeline-stands`, cleared at the step's own `step-ended` or its next sign of life. A step waiting on a person or
|
||||
// queued for a build node (§86) is owed no progress. At its start the watchman seeds its open steps from the runner's `stands`.
|
||||
|
||||
// stepRow is the signals table's row of a step standing still.
|
||||
const stepRow = "step-stands"
|
||||
|
||||
// openStep is a step started and not ended.
|
||||
type openStep struct {
|
||||
pipeline, step, artifact string
|
||||
// asks are the step's own ask and the asks its shared runs name; ask the one last heard of.
|
||||
asks map[string]bool
|
||||
ask string
|
||||
// at is when the clock began; owed says whether the step is owed progress.
|
||||
at time.Time
|
||||
owed bool
|
||||
}
|
||||
|
||||
// stepKey is the key an open step is kept under.
|
||||
func stepKey(pipeline, step, artifact string) string { return pipeline + "/" + step + "/" + artifact }
|
||||
|
||||
// StepFindingID is the finding's id of an open step.
|
||||
func StepFindingID(key string) string { return "step-" + unsafeKey.ReplaceAllString(key, "_") }
|
||||
|
||||
// StepStarted hears the runner's `step-started`: the step is open and its clock begins at the event's time, unless it waits on a
|
||||
// person or for a build node, which are owed no progress. One that is no running or waiting step is ended.
|
||||
func (w *Watchman) StepStarted(body []byte) {
|
||||
var e meshpipelinerunner.StepEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
|
||||
w.say("a step-started does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
key := stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
switch e.Result.Outcome {
|
||||
case meshpipelinerunner.StepResultOutcomeRunning, "", meshpipelinerunner.StepResultOutcomeWaiting,
|
||||
meshpipelinerunner.StepResultOutcomeWaitingForABuildNode:
|
||||
default:
|
||||
w.closeStep(key)
|
||||
return
|
||||
}
|
||||
at := w.now()
|
||||
switch {
|
||||
case e.At != nil && !e.At.IsZero():
|
||||
at = *e.At
|
||||
case e.Result.Started != nil && !e.Result.Started.IsZero():
|
||||
at = *e.Result.Started
|
||||
}
|
||||
owed := e.Result.Outcome == meshpipelinerunner.StepResultOutcomeRunning || e.Result.Outcome == ""
|
||||
w.openStep(key, e.Pipeline, e.Result.Step, e.Result.Artifact, e.Result.Ask, sharedAsks(e.Result.Shared), at, owed)
|
||||
}
|
||||
|
||||
// StepEnded hears the runner's `step-ended`: the step's clock ends and its finding is cleared.
|
||||
func (w *Watchman) StepEnded(body []byte) {
|
||||
var e meshpipelinerunner.StepEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
|
||||
w.say("a step-ended does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.closeStep(stepKey(e.Pipeline, e.Result.Step, e.Result.Artifact))
|
||||
}
|
||||
|
||||
// WantsStepProgress hears the seat mesh-wants' `step-progress` (the assigner's, of a composition step): a step whose ask it is, or
|
||||
// a shared ask it names, is alive; the runs it names are kept as the step's.
|
||||
func (w *Watchman) WantsStepProgress(body []byte) {
|
||||
var e meshwants.StepProgressEvent
|
||||
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
|
||||
w.say("a step-progress of mesh-wants does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
var named []string
|
||||
for _, s := range e.Shared {
|
||||
if s.Ask != "" {
|
||||
named = append(named, s.Ask)
|
||||
}
|
||||
}
|
||||
for key, s := range w.steps {
|
||||
if !s.asks[e.Ask] {
|
||||
continue
|
||||
}
|
||||
for _, a := range named {
|
||||
s.asks[a] = true
|
||||
}
|
||||
w.stepAlive(key, e.Ask, e.At)
|
||||
}
|
||||
}
|
||||
|
||||
// stepsHeardOf says the steps that name ask are alive: a build seat said its progress or its end (§732).
|
||||
func (w *Watchman) stepsHeardOf(ask string, at time.Time) {
|
||||
if ask == "" {
|
||||
return
|
||||
}
|
||||
for key, s := range w.steps {
|
||||
if s.asks[ask] {
|
||||
w.stepAlive(key, ask, at)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stepAlive begins an owed step's clock again at at, never earlier than it began, w.mu held.
|
||||
func (w *Watchman) stepAlive(key, ask string, at time.Time) {
|
||||
s := w.steps[key]
|
||||
if at.IsZero() {
|
||||
at = w.now()
|
||||
}
|
||||
s.ask = ask
|
||||
if at.After(s.at) {
|
||||
s.at = at
|
||||
}
|
||||
w.timeStep(key)
|
||||
}
|
||||
|
||||
func sharedAsks(shared []meshpipelinerunner.Shared) []string {
|
||||
var out []string
|
||||
for _, s := range shared {
|
||||
if s.Ask != "" {
|
||||
out = append(out, s.Ask)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// openStep keeps a step and times it, w.mu held. A step kept before keeps the asks it knew.
|
||||
func (w *Watchman) openStep(key, pipeline, step, artifact, ask string, shared []string, at time.Time, owed bool) {
|
||||
s := w.steps[key]
|
||||
if s == nil {
|
||||
s = &openStep{pipeline: pipeline, step: step, artifact: artifact, asks: map[string]bool{}}
|
||||
w.steps[key] = s
|
||||
}
|
||||
s.at, s.owed = at, owed
|
||||
if ask != "" {
|
||||
s.asks[ask], s.ask = true, ask
|
||||
}
|
||||
for _, a := range shared {
|
||||
s.asks[a] = true
|
||||
}
|
||||
w.timeStep(key)
|
||||
}
|
||||
|
||||
// closeStep ends a step, w.mu held.
|
||||
func (w *Watchman) closeStep(key string) {
|
||||
if _, ok := w.steps[key]; !ok {
|
||||
return
|
||||
}
|
||||
delete(w.steps, key)
|
||||
id := StepFindingID(key)
|
||||
w.unsignal(stepRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
}
|
||||
|
||||
// timeStep times an open step, w.mu held: the progress bound from its clock, the finding raised at once when that has passed.
|
||||
func (w *Watchman) timeStep(key string) {
|
||||
s := w.steps[key]
|
||||
id := StepFindingID(key)
|
||||
w.unsignal(stepRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
if !s.owed {
|
||||
return
|
||||
}
|
||||
name := s.pipeline + " " + s.step
|
||||
if s.artifact != "" {
|
||||
name += " " + s.artifact
|
||||
}
|
||||
last := ""
|
||||
if s.ask != "" {
|
||||
last = ", last heard of its ask " + s.ask
|
||||
}
|
||||
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: stepRow, Since: s.at,
|
||||
Summary: fmt.Sprintf("the step %s has said nothing since %s%s", name, stamp(s.at), last),
|
||||
Said: "no progress in " + w.progressBound.String(),
|
||||
Headline: "A pipeline step has stopped saying it lives",
|
||||
Explanation: "The step " + name + " started and has not ended, and neither it nor the run it reads has said its progress for " +
|
||||
w.progressBound.String() + " or more" + last + ".",
|
||||
Needs: "the part that runs it to say its progress, or the step ended (the runner's verbs mesh-pipeline-runner.stands and mesh-pipeline-runner.cancel)"}
|
||||
if left := w.progressBound - w.now().Sub(s.at); left > 0 {
|
||||
w.signal(stepRow, id, left, signalDue{Do: dueFind, Finding: f})
|
||||
return
|
||||
}
|
||||
w.find(f)
|
||||
}
|
||||
|
||||
// SeedSteps seeds the open steps from the runner's `stands` at the watchman's start (§731, A-468j): each step not already kept
|
||||
// begins its clock now, and a step that waits on a person or for a build node is owed none.
|
||||
func (w *Watchman) SeedSteps(stands []meshpipelinerunner.Wait) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for _, s := range stands {
|
||||
if s.Pipeline == "" || s.Step == "" {
|
||||
continue
|
||||
}
|
||||
key := stepKey(s.Pipeline, s.Step, s.Artifact)
|
||||
if _, ok := w.steps[key]; ok {
|
||||
continue
|
||||
}
|
||||
owed := s.On != meshpipelinerunner.WaitOnPerson && s.On != meshpipelinerunner.WaitOnBuildNode && s.Who == ""
|
||||
w.openStep(key, s.Pipeline, s.Step, s.Artifact, s.Ask, sharedAsks(s.Shared), w.now(), owed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
func stepBody(pipeline, step, artifact, ask, outcome string, at time.Time, shared ...string) []byte {
|
||||
var sh []meshpipelinerunner.Shared
|
||||
for _, a := range shared {
|
||||
sh = append(sh, meshpipelinerunner.Shared{Run: "r", Ask: a})
|
||||
}
|
||||
b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c", At: &at,
|
||||
Result: meshpipelinerunner.StepResult{Step: step, Artifact: artifact, Ask: ask, Outcome: outcome, Shared: sh}})
|
||||
return b
|
||||
}
|
||||
|
||||
func wantsProgress(ask string, at time.Time, shared ...string) []byte {
|
||||
var sh []meshwants.Shared
|
||||
for _, a := range shared {
|
||||
sh = append(sh, meshwants.Shared{Machine: "n1", Run: "r", Ask: a})
|
||||
}
|
||||
b, _ := json.Marshal(meshwants.StepProgressEvent{Ask: ask, Asker: "a", At: at, Phase: "run", Shared: sh})
|
||||
return b
|
||||
}
|
||||
|
||||
// design 47 G §731, §732, A-468j, A-469, A-469o, A-469p (test 108's and 109's watchman case): a step started and not ended is
|
||||
// timed from its `step-started` and from every sign of life of its own ask or an ask its `shared` names, raises `pipeline-stands`
|
||||
// at the five-minute bound and clears at its `step-ended`; a step on a person or queued for a build node is owed none; the open
|
||||
// steps are seeded from `stands`.
|
||||
func TestAStepThatReadsARunIsTimed(t *testing.T) {
|
||||
run := meshpipelinerunner.StepResultOutcomeRunning
|
||||
key := func(pipeline, step, artifact string) string {
|
||||
return signalKey(stepRow, StepFindingID(stepKey(pipeline, step, artifact)))
|
||||
}
|
||||
finding := func(pipeline, step, artifact string) string {
|
||||
return FindingKey(ScopeDelivery, StepFindingID(stepKey(pipeline, step, artifact)), KindPipelineStands)
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
run func(w *Watchman, o *fakeOut, c *clock)
|
||||
// stands are the finding keys that stand at the end.
|
||||
stands []string
|
||||
}{
|
||||
{"no progress for five minutes", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
|
||||
if o.ttls[key("p", "composition", "")] != ProgressBound || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
expire(w, c, key("p", "composition", ""), ProgressBound)
|
||||
}, []string{finding("p", "composition", "")}},
|
||||
{"the assigner's progress of its ask begins the clock again, and names the runs", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.WantsStepProgress(wantsProgress("a1", c.t, "run1"))
|
||||
if o.ttls[key("p", "composition", "")] != ProgressBound {
|
||||
t.Fatalf("clock not begun again: %v", o.ttls)
|
||||
}
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.StepProgress("node-build-go", "n1", askBody("run1", c.t)) // an ask the progress named
|
||||
if o.ttls[key("p", "composition", "")] != ProgressBound {
|
||||
t.Fatalf("a named run's progress is the step's: %v", o.ttls)
|
||||
}
|
||||
w.WantsStepProgress(wantsProgress("other", c.t))
|
||||
}, nil},
|
||||
{"the progress of a shared run's ask, and its end, begin a test step's clock again", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t, "run-ask"))
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.StepProgress("node-build-go", "n1", askBody("run-ask", c.t))
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.AskEnded("node-build-go", askBody("run-ask", c.t))
|
||||
expire(w, c, key("p", "unit", "m:program"), ProgressBound)
|
||||
}, []string{finding("p", "unit", "m:program")}},
|
||||
{"the artifact keys the step: the other artifact's end clears only its own", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t))
|
||||
w.StepStarted(stepBody("p", "unit", "m:tools", "a3", run, c.t))
|
||||
c.t = c.t.Add(ProgressBound)
|
||||
w.Pulse(key("p", "unit", "m:program"), Expired, c.t)
|
||||
w.Pulse(key("p", "unit", "m:tools"), Expired, c.t)
|
||||
w.StepEnded(stepBody("p", "unit", "m:tools", "a3", meshpipelinerunner.StepResultOutcomePassed, c.t))
|
||||
}, []string{finding("p", "unit", "m:program")}},
|
||||
{"its step-ended clears the finding", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "composition", "", "a1", run, c.t))
|
||||
expire(w, c, key("p", "composition", ""), ProgressBound)
|
||||
w.StepEnded(stepBody("p", "composition", "", "a1", meshpipelinerunner.StepResultOutcomeFailed, c.t))
|
||||
}, nil},
|
||||
{"a step on a person or queued for a build node is owed none", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.StepStarted(stepBody("p", "approval", "", "", meshpipelinerunner.StepResultOutcomeWaiting, c.t))
|
||||
w.StepStarted(stepBody("p", "unit", "m:program", "a2", meshpipelinerunner.StepResultOutcomeWaitingForABuildNode, c.t))
|
||||
c.t = c.t.Add(time.Hour)
|
||||
if len(o.pulses) != 0 || len(o.findings) != 0 {
|
||||
t.Fatalf("owed none: %v %v", o.pulses, o.findings)
|
||||
}
|
||||
w.StepStarted(stepBody("p", "unit", "m:program", "a2", run, c.t)) // taken: now owed
|
||||
expire(w, c, key("p", "unit", "m:program"), ProgressBound)
|
||||
}, []string{finding("p", "unit", "m:program")}},
|
||||
{"seeded from stands: each clock begins at the start, a step on a person is owed none", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SeedSteps([]meshpipelinerunner.Wait{
|
||||
{Pipeline: "p", Step: "composition", Ask: "a1", On: ""},
|
||||
{Pipeline: "p", Step: "unit", Artifact: "m:program", Shared: []meshpipelinerunner.Shared{{Run: "r", Ask: "run-ask"}}},
|
||||
{Pipeline: "p", Step: "approval", On: meshpipelinerunner.WaitOnPerson},
|
||||
})
|
||||
if len(o.findings) != 0 || o.ttls[key("p", "composition", "")] != ProgressBound || len(o.pulses) != 2 {
|
||||
t.Fatalf("seeded at the start: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
c.t = c.t.Add(4 * time.Minute)
|
||||
w.WantsStepProgress(wantsProgress("a1", c.t)) // the composition step lives
|
||||
expire(w, c, key("p", "unit", "m:program"), time.Minute)
|
||||
}, []string{finding("p", "unit", "m:program")}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
w.SetStepBound(ProgressBound) // setup shortens it for the older tests
|
||||
tc.run(w, o, c)
|
||||
if len(o.findings) != len(tc.stands) {
|
||||
t.Fatalf("findings %v, want %v", o.findings, tc.stands)
|
||||
}
|
||||
for _, k := range tc.stands {
|
||||
if _, ok := o.findings[k]; !ok {
|
||||
t.Fatalf("findings %v, want %s", o.findings, k)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -121,6 +121,8 @@ type Watchman struct {
|
||||
recordMilestone map[string]string
|
||||
// queued is, by build seat, each step queued on it for a build node and since when (pipelines.go).
|
||||
queued map[string]map[string]time.Time
|
||||
// steps is each pipeline step started and not ended, by `<pipeline>/<step>/<artifact>` (steps.go, design 47 G §732).
|
||||
steps map[string]*openStep
|
||||
// awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go).
|
||||
awaySaid map[string]time.Time
|
||||
// opened is each record past its bound and not ended, with its finding (records.go).
|
||||
@@ -135,7 +137,7 @@ func New(module string, out Out, now func() time.Time) *Watchman {
|
||||
advised: map[string]int{}, advisedSince: map[string]time.Time{},
|
||||
provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
|
||||
|
||||
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
|
||||
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, steps: map[string]*openStep{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
|
||||
}
|
||||
|
||||
// Logf sets where a write that failed is said; it is silent until set.
|
||||
|
||||
@@ -127,11 +127,11 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
// The join: its first view asked again by hand is a hand act, recorded through the seat (views.StatusTools). The waits the
|
||||
// indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's.
|
||||
if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably.
|
||||
if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) {
|
||||
if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
// 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4).
|
||||
|
||||
@@ -26,7 +26,8 @@
|
||||
"seat:mesh-broker.show",
|
||||
"seat:mesh-catalog.revision",
|
||||
"seat:mesh-catalog.registrations",
|
||||
"seat:mesh-delivery.source"
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:mesh-pipeline-runner.stands"
|
||||
],
|
||||
"consumes": [
|
||||
"seat:mesh-wants.views.wants",
|
||||
@@ -64,6 +65,7 @@
|
||||
"mesh-pipeline-runner.pipeline-started",
|
||||
"mesh-pipeline-runner.step-started",
|
||||
"mesh-pipeline-runner.step-ended",
|
||||
"mesh-wants.step-progress",
|
||||
"mesh-pipeline-runner.wait-held",
|
||||
"mesh-pipeline-runner.wait-ended",
|
||||
"mesh-wants.wait-held",
|
||||
|
||||
Reference in New Issue
Block a user