Compare commits
14
Commits
go/v0.1.66
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b0082ec04f | ||
|
|
6b6e4c81f0 | ||
|
|
374d060cc0 | ||
|
|
1f878eec58 | ||
|
|
c9515da86f | ||
|
|
5f7137e676 | ||
|
|
902947a044 | ||
|
|
b95b3ef989 | ||
|
|
0f167db73d | ||
|
|
e5f54c5cdb | ||
|
|
bdb4edd820 | ||
|
|
31c2c5c7fe | ||
|
|
ad0052c6f5 | ||
|
|
aef8632d95 |
@@ -289,7 +289,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The indexer's findings are the seat mesh-conditions' generated finding, written through mesh-sdk's held finding writer
|
||||
// (design 47 G §689, A-29): it holds what is written before the state is bound, and starts once the state is.
|
||||
writer, err := views.NewHeldWriter(views.WriterConfig[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]{
|
||||
Module: cred.Module, State: calls.State(index.FindingsState), Node: cred.Node, Asked: mcseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logger.Printf})
|
||||
Module: cred.Module, State: calls.State(index.FindingsState), Node: cred.Node, Asked: mcseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logger.Printf,
|
||||
// its own kinds only (G §580, H-A-242, H-A-247): the tool runner's writer states its kinds over the same state
|
||||
Kinds: index.FindingKinds, Retired: index.RetiredFindingKinds})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the indexer's finding writer cannot be made: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// Design 47 H H-A-240, H-A-242, H-A-247, G §580: every finding kind the daemon's code raises or clears is one its writer owns
|
||||
// (FindingKinds, and a views reader's views.ViewKinds), so the writer never refuses a kind of the indexer's own. The test
|
||||
// reads the daemon's code (this package and cmd/mesh-delivery-indexer) for every place a kind is given to the writer: the
|
||||
// kind argument of observed, finding, clearFinding, clearHeld and a writer's Clear, the Kind of a Finding literal, and the
|
||||
// kinds of an SDK part handed the writer (partKinds). mcc.KindCallUnanswered stays readable, so a return to the tool runner's
|
||||
// kind fails the test. A
|
||||
// kind there is a string, a constant of the package or a constant named in imported below; any other form fails the test,
|
||||
// naming its place, but for the pass-through parameter `kind`, a held finding's `.Kind` and the kind cut from a key a views
|
||||
// reader names, which carry a kind given elsewhere.
|
||||
func TestFindingKindsHoldEveryKindTheCodeRaises(t *testing.T) {
|
||||
imported := map[string]string{"mcc.KindCallUnanswered": mcc.KindCallUnanswered, "views.ViewWaitsKind": views.ViewWaitsKind}
|
||||
// kindArg is the place of the kind among a raising call's arguments.
|
||||
kindArg := map[string]int{"observed": 2, "finding": 2, "clearFinding": 3, "clearHeld": 4, "Clear": 3}
|
||||
// partKinds are the kinds each SDK part raises through the writer it is handed (H-A-247 (1)); handed says a literal's field
|
||||
// gives it the writer.
|
||||
partKinds := map[string][]string{"views.ProcessReader": {views.EventUnhandledKind}, "views.Emitter": {views.UnsaidKind, views.TooLargeKind},
|
||||
"heard.Writer": {heard.Kind}}
|
||||
handed := func(e ast.Expr) bool {
|
||||
kv, ok := e.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
id, ok := kv.Key.(*ast.Ident)
|
||||
return ok && (id.Name == "Findings" || id.Name == "Put")
|
||||
}
|
||||
fset := token.NewFileSet()
|
||||
var files []*ast.File
|
||||
consts := map[string]string{}
|
||||
for _, dir := range []string{".", filepath.Join("..", "..", "cmd", "mesh-delivery-indexer")} {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !strings.HasSuffix(e.Name(), ".go") || strings.HasSuffix(e.Name(), "_test.go") {
|
||||
continue
|
||||
}
|
||||
f, err := parser.ParseFile(fset, filepath.Join(dir, e.Name()), nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files = append(files, f)
|
||||
if dir != "." {
|
||||
continue
|
||||
}
|
||||
for _, d := range f.Decls {
|
||||
g, ok := d.(*ast.GenDecl)
|
||||
if !ok || g.Tok != token.CONST {
|
||||
continue
|
||||
}
|
||||
for _, s := range g.Specs {
|
||||
v := s.(*ast.ValueSpec)
|
||||
for i, n := range v.Names {
|
||||
if i < len(v.Values) {
|
||||
if l, ok := v.Values[i].(*ast.BasicLit); ok && l.Kind == token.STRING {
|
||||
consts[n.Name], _ = strconv.Unquote(l.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
declared := append(slices.Clone(FindingKinds), views.ViewKinds...)
|
||||
raised := map[string]string{} // kind -> a place it is raised
|
||||
kindOf := func(e ast.Expr) (string, bool) {
|
||||
switch x := e.(type) {
|
||||
case *ast.BasicLit:
|
||||
s, err := strconv.Unquote(x.Value)
|
||||
return s, err == nil
|
||||
case *ast.Ident:
|
||||
if x.Name == "kind" {
|
||||
return "", true // a pass-through parameter: its callers give the kind
|
||||
}
|
||||
s, ok := consts[x.Name]
|
||||
if !ok {
|
||||
s, ok = imported["index."+x.Name]
|
||||
}
|
||||
return s, ok
|
||||
case *ast.SliceExpr:
|
||||
return "", true // the kind of a key a views reader names: its own kinds (views.ViewKinds)
|
||||
case *ast.SelectorExpr:
|
||||
if x.Sel.Name == "Kind" {
|
||||
return "", true // a held finding's kind, given when it was raised
|
||||
}
|
||||
if p, ok := x.X.(*ast.Ident); ok {
|
||||
if p.Name == "index" {
|
||||
s, ok := consts[x.Sel.Name]
|
||||
return s, ok
|
||||
}
|
||||
s, ok := imported[p.Name+"."+x.Sel.Name]
|
||||
return s, ok
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
note := func(e ast.Expr) {
|
||||
k, ok := kindOf(e)
|
||||
if !ok {
|
||||
t.Errorf("%s: a finding kind the test cannot read; give it as a constant (or name it in imported)", fset.Position(e.Pos()))
|
||||
return
|
||||
}
|
||||
if k != "" {
|
||||
raised[k] = fset.Position(e.Pos()).String()
|
||||
}
|
||||
}
|
||||
for _, f := range files {
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch x := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
var name string
|
||||
switch fn := x.Fun.(type) {
|
||||
case *ast.Ident:
|
||||
name = fn.Name
|
||||
case *ast.SelectorExpr:
|
||||
name = fn.Sel.Name
|
||||
}
|
||||
if i, ok := kindArg[name]; ok && len(x.Args) > i {
|
||||
if name == "Clear" && len(x.Args) != 5 {
|
||||
return true // not a finding writer's Clear(ctx, scope, id, kind, why)
|
||||
}
|
||||
note(x.Args[i])
|
||||
}
|
||||
case *ast.CompositeLit:
|
||||
// an SDK part handed the writer raises its own kinds through it (H-A-247 (1))
|
||||
if s, ok := x.Type.(*ast.SelectorExpr); ok && slices.ContainsFunc(x.Elts, handed) {
|
||||
if p, ok := s.X.(*ast.Ident); ok && p.Name != "index" { // the indexer's own code is read above
|
||||
part := p.Name + "." + s.Sel.Name
|
||||
if kinds, ok := partKinds[part]; ok {
|
||||
for _, k := range kinds {
|
||||
raised[k] = fset.Position(x.Pos()).String() + " (" + part + ")"
|
||||
}
|
||||
} else {
|
||||
t.Errorf("%s: %s is handed the finding writer, and the test does not know its kinds: name them in partKinds",
|
||||
fset.Position(x.Pos()), part)
|
||||
}
|
||||
}
|
||||
}
|
||||
if s, ok := x.Type.(*ast.SelectorExpr); ok && s.Sel.Name == "Finding" {
|
||||
for _, el := range x.Elts {
|
||||
if kv, ok := el.(*ast.KeyValueExpr); ok {
|
||||
if id, ok := kv.Key.(*ast.Ident); ok && id.Name == "Kind" {
|
||||
note(kv.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
for _, k := range []string{PinMissingKind, OutOfStepKind, PackagesNotPublishedKind, CallUnansweredKind, views.EventUnhandledKind, heard.Kind} {
|
||||
if _, ok := raised[k]; !ok {
|
||||
t.Errorf("the test no longer finds the kind %s the code raises: it reads the code wrongly", k)
|
||||
}
|
||||
}
|
||||
for k, at := range raised {
|
||||
if !slices.Contains(declared, k) {
|
||||
t.Errorf("%s: the code raises the finding kind %q, which the writer's list FindingKinds does not declare", at, k)
|
||||
}
|
||||
}
|
||||
// a retired kind is one no code raises and the writer does not own (H-A-247 (3))
|
||||
for _, r := range RetiredFindingKinds {
|
||||
if at, ok := raised[r.Kind]; ok || slices.Contains(declared, r.Kind) {
|
||||
t.Errorf("the retired kind %q is still declared or raised (%s)", r.Kind, at)
|
||||
}
|
||||
}
|
||||
// H-A-249: the retired call-unanswered names the ids the indexer's unanswered calls are written under, as in a key
|
||||
covers := func(scope, id string) bool {
|
||||
for _, r := range RetiredFindingKinds {
|
||||
if r.Kind == mcc.KindCallUnanswered && (r.Scope == "" || r.Scope == scope) && (slices.Contains(r.IDs, id) ||
|
||||
slices.ContainsFunc(r.Prefixes, func(p string) bool { return strings.HasPrefix(id, p) })) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, id := range []string{keyID("node-build-go." + StepsVerb), keyID("node-build-image." + StepsVerb), keyID(floorKey("mesh-tools")),
|
||||
keyID(BuilderVersionsCall)} {
|
||||
if !covers(mcc.ScopeCall, id) {
|
||||
t.Errorf("the retired call-unanswered does not cover the indexer's id %s", id)
|
||||
}
|
||||
}
|
||||
// the approval gate's findings: scope machine, id <node>=2e<ask>
|
||||
if covers("machine", "novox=2eask-1") || covers("machine", "node-build-go.steps") || covers(mcc.ScopeCall, "novox=2eask-1") {
|
||||
t.Error("the retired call-unanswered covers an id of the tool runner's")
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// FindingsState is the indexer's state of its findings (design 47 G §383, §687).
|
||||
@@ -12,6 +14,26 @@ const (
|
||||
FindingsState = "findings"
|
||||
)
|
||||
|
||||
// CallUnansweredKind is the indexer's finding of a call nobody answered within its bound (scope call, its id the call): a kind
|
||||
// of its own, since the tool runner's writer over the same state owns call-unanswered (design 47 H H-A-247 (2)).
|
||||
const CallUnansweredKind = "indexer-call-unanswered"
|
||||
|
||||
// FindingKinds are the finding kinds the indexer's daemon raises, the kinds its writer owns (design 47 G §580, H H-A-240,
|
||||
// H-A-242, H-A-247): its own, the process reader's event-unhandled and the family-not-heard of the heard writer it is handed
|
||||
// (H-A-247 (1)); a views reader's and the emitter's (views.ViewKinds) are always the writer's besides. The tool runner's
|
||||
// writer over the same state `findings` owns its own. TestFindingKindsHoldEveryKindTheCodeRaises holds the code to this list.
|
||||
var FindingKinds = []string{PinMissingKind, OutOfStepKind, PackagesNotPublishedKind, HistoryFillingKind, SeedWaitsKind,
|
||||
SeedLeftOutKind, SeatDefinitionUnparsedKind, SeatDefinedTwiceKind, SeatOutsideItsDomainKind, CallUnansweredKind,
|
||||
views.EventUnhandledKind, heard.Kind}
|
||||
|
||||
// RetiredFindingKinds are kinds no code of the indexer raises any more, which its writer clears once at its start (H-A-247
|
||||
// (3)): merge-untaken went with H-A-82; call-unanswered became indexer-call-unanswered (H-A-247 (2)), and since the tool
|
||||
// runner's writer owns call-unanswered in the same state, only the ids the indexer wrote under it, in scope call, are cleared (H-A-249):
|
||||
// the seed's <build seat>.steps (every build seat is named node-build-<kind>), the floor's mesh-catalog.versions-floor.<module>
|
||||
// and the versions fill's mesh-delivery-builder.builder_versions.
|
||||
var RetiredFindingKinds = []views.RetiredKind{{Kind: "merge-untaken"}, {Kind: mcc.KindCallUnanswered, Scope: mcc.ScopeCall,
|
||||
IDs: []string{keyID(BuilderVersionsCall)}, Prefixes: []string{"node-build-", keyID(FloorCall) + "."}}}
|
||||
|
||||
// FindingWriter is the indexer's finding writer, mesh-sdk's held writer of the seat mesh-conditions' generated finding (design
|
||||
// 47 G A-29, A-38): Write raises a finding, Clear clears one, Held answers the findings it holds, so the indexer reads what
|
||||
// stands from the writer.
|
||||
|
||||
@@ -124,7 +124,7 @@ func (r *Registry) ProposeSeed(ctx context.Context, modules []SeedModule, again
|
||||
return "", err
|
||||
}
|
||||
if len(seed.Unanswered) > 0 {
|
||||
return "", fmt.Errorf("the build seats %v have not answered steps: the seed waits for them (call-unanswered after five minutes)", seed.Unanswered)
|
||||
return "", fmt.Errorf("the build seats %v have not answered steps: the seed waits for them (indexer-call-unanswered after five minutes)", seed.Unanswered)
|
||||
}
|
||||
regs, left = seed.Registrations, seed.Left
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (st
|
||||
// SeedModulesFrom reads the modules the newest trunk revision holds for the seed (design 47 G §704, §8): for each artifact of
|
||||
// each module the seat list's build seat answers steps with {module, artifact, fingerprint, fetch} from the module's trunk
|
||||
// entry, the fetch its repository, commit, directory and the artifact's named paths. A seat that does not answer leaves
|
||||
// its artifact unanswered, which holds the seed; its call-unanswered finding (scope call, id <seat>.steps) is written once
|
||||
// its artifact unanswered, which holds the seed; its indexer-call-unanswered finding (scope call, id <seat>.steps) is written once
|
||||
// it is unanswered five minutes and cleared when it answers.
|
||||
func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.Definition, steps StepsCaller) ([]SeedModule, error) {
|
||||
if ix.Revisions == nil {
|
||||
@@ -202,7 +202,7 @@ func artifactName(ref string) string {
|
||||
return ref
|
||||
}
|
||||
|
||||
// stepsFindings writes call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
|
||||
// stepsFindings writes indexer-call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
|
||||
func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, unanswered map[string]error) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
@@ -223,11 +223,11 @@ func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool,
|
||||
}
|
||||
for _, seat := range slices.Sorted(slices.Values(raise)) {
|
||||
id := seat + "." + StepsVerb
|
||||
f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered,
|
||||
f := ix.observed(mcc.ScopeCall, id, CallUnansweredKind,
|
||||
fmt.Sprintf("the call %s is unanswered since %s", id, since[seat].UTC().Format(time.RFC3339)),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The seed asks %s of every artifact its build seat builds; it is not answered since %s (%v), so the first registrations wait for it.", id, since[seat].UTC().Format(time.RFC3339), unanswered[seat]),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
"indexer."+CallUnansweredKind)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("seed: the finding %s is not written: %v", id, err)
|
||||
}
|
||||
@@ -236,7 +236,7 @@ func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool,
|
||||
if _, bad := unanswered[seat]; bad {
|
||||
continue
|
||||
}
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, mcc.KindCallUnanswered, "the seat answers steps"); err != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, CallUnansweredKind, "the seat answers steps"); err != nil {
|
||||
ix.logf("seed: the finding %s.%s is not cleared: %v", seat, StepsVerb, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
// Design 47 G §704: the seed asks `steps` of the build seat whose `build` rule matches each artifact, with {module,
|
||||
// artifact, fingerprint, fetch} from the newest trunk entry; a seat that does not answer holds the seed and, after five
|
||||
// minutes, is the finding call-unanswered (scope call, id <seat>.steps), cleared when it answers.
|
||||
// minutes, is the finding indexer-call-unanswered (scope call, id <seat>.steps), cleared when it answers.
|
||||
func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
@@ -85,7 +85,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
|
||||
if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := w.Held(); len(h) != 1 || h[0].Scope != mcc.ScopeCall || h[0].ID != "node-build-go.steps" || h[0].Kind != mcc.KindCallUnanswered {
|
||||
if h := w.Held(); len(h) != 1 || h[0].Scope != mcc.ScopeCall || h[0].ID != "node-build-go.steps" || h[0].Kind != CallUnansweredKind {
|
||||
t.Fatalf("the findings after five minutes: %+v", h)
|
||||
}
|
||||
goUp = true
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
// filled per module whenever the indexer starts and at each catalog revision kept, not once (A-490d, A-490f): the key
|
||||
// versions-filled closes only the builder's half of the fill.
|
||||
|
||||
// FloorCall is the id of the finding call-unanswered when a floor read (the forge's tags or a package holder's versions) is
|
||||
// FloorCall is the id of the finding indexer-call-unanswered when a floor read (the forge's tags or a package holder's versions) is
|
||||
// unanswered five minutes after the first (A-490h).
|
||||
const FloorCall = "mesh-catalog.versions-floor"
|
||||
|
||||
@@ -91,7 +91,7 @@ func (ix *Indexer) FloorModules(ctx context.Context) ([]string, error) {
|
||||
// FillFloor fills one module's floor, the highest version among its packages' tags and holders' versions (A-490g). Its reads are
|
||||
// kept calls made through mesh-sdk's form, which asks again through every transport fault; a read it gives up on (its own
|
||||
// pause spent) is asked again at once, never given up, so the read has no bound (A-490i). The first unanswered try starts the
|
||||
// module's clock: five minutes later the finding call-unanswered is written, and cleared when a read answers. A read the
|
||||
// module's clock: five minutes later the finding indexer-call-unanswered is written, and cleared when a read answers. A read the
|
||||
// callee answered with a failure ends the fill, left for the next start or revision. One fill of a module runs at a time.
|
||||
func (ix *Indexer) FillFloor(ctx context.Context, module string, src FloorSources) error {
|
||||
if _, running := ix.floors.LoadOrStore(module, true); running {
|
||||
@@ -198,7 +198,7 @@ func FloorSinceKey(module string) string { return floorSinceKey(module) }
|
||||
|
||||
func floorSinceKey(module string) string { return "call-since." + keys.EscapeName(floorKey(module)) }
|
||||
|
||||
// floorUnanswered writes call-unanswered once the module's floor read is unanswered five minutes (the time kept in the own state).
|
||||
// floorUnanswered writes indexer-call-unanswered once the module's floor read is unanswered five minutes (the time kept in the own state).
|
||||
func (ix *Indexer) floorUnanswered(ctx context.Context, module string, first time.Time, cause error) {
|
||||
now := ix.now()
|
||||
since := ix.unansweredSince(ctx, floorSinceKey(module), first)
|
||||
@@ -206,11 +206,11 @@ func (ix *Indexer) floorUnanswered(ctx context.Context, module string, first tim
|
||||
return
|
||||
}
|
||||
at := since.UTC().Format(time.RFC3339)
|
||||
f := ix.observed(mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered,
|
||||
f := ix.observed(mcc.ScopeCall, floorKey(module), CallUnansweredKind,
|
||||
fmt.Sprintf("the floor of %s's versions is unanswered since %s", module, at),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The floor of %s's versions reads the forge's tags and the package holders' versions; a read is not answered since %s (%v), so it has no version.", module, at, cause),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
"indexer."+CallUnansweredKind)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("floor: the finding %s is not written: %v", floorKey(module), err)
|
||||
}
|
||||
@@ -219,7 +219,7 @@ func (ix *Indexer) floorUnanswered(ctx context.Context, module string, first tim
|
||||
func (ix *Indexer) floorAnswered(ctx context.Context, module string) {
|
||||
ix.answered(ctx, floorSinceKey(module))
|
||||
if ix.Findings != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered, "the floor's reads answer"); err != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, floorKey(module), CallUnansweredKind, "the floor's reads answer"); err != nil {
|
||||
ix.logf("floor: the finding %s is not cleared: %v", floorKey(module), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ func versionNameOf(module string, a meshcatalog.Artifact) string {
|
||||
|
||||
// FillVersions fills the state versions once from builder_versions and says versions-filled (§372). A fill already made
|
||||
// is not made again; a builder_versions that does not answer holds the fill and, after five minutes, is the finding
|
||||
// call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers.
|
||||
// indexer-call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers.
|
||||
func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
if ix.Versions == nil || ix.Heard == nil {
|
||||
return errors.New("the state versions is not kept here")
|
||||
@@ -127,7 +127,7 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
}
|
||||
ix.answered(ctx, versionsFillSince)
|
||||
if ix.Findings != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered, "builder_versions answers"); err != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, BuilderVersionsCall, CallUnansweredKind, "builder_versions answers"); err != nil {
|
||||
ix.logf("versions fill: the finding %s is not cleared: %v", BuilderVersionsCall, err)
|
||||
}
|
||||
}
|
||||
@@ -184,18 +184,18 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillUnanswered writes call-unanswered once builder_versions is unanswered five minutes (the time kept in the own state).
|
||||
// fillUnanswered writes indexer-call-unanswered once builder_versions is unanswered five minutes (the time kept in the own state).
|
||||
func (ix *Indexer) fillUnanswered(ctx context.Context, cause error) {
|
||||
now := ix.now()
|
||||
since := ix.unansweredSince(ctx, versionsFillSince, now)
|
||||
if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter {
|
||||
return
|
||||
}
|
||||
f := ix.observed(mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered,
|
||||
f := ix.observed(mcc.ScopeCall, BuilderVersionsCall, CallUnansweredKind,
|
||||
fmt.Sprintf("the call %s is unanswered since %s", BuilderVersionsCall, since.UTC().Format(time.RFC3339)),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The first fill of the catalog's versions asks %s of every artifact; it is not answered since %s (%v), so no version is released and the builder reserves nothing.", BuilderVersionsCall, since.UTC().Format(time.RFC3339), cause),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
"indexer."+CallUnansweredKind)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("versions fill: the finding %s is not written: %v", BuilderVersionsCall, err)
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
|
||||
|
||||
// Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or
|
||||
// published, over its artifacts as released and says versions-filled once; a builder_versions that does not answer holds it and, after five minutes, is the finding
|
||||
// call-unanswered; a fill made is not made again.
|
||||
// indexer-call-unanswered; a fill made is not made again.
|
||||
func TestTheFirstFillOfTheVersionsComesFromTheBuildersNumbers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
@@ -501,7 +501,7 @@ func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) {
|
||||
|
||||
// Design 47 G §412 as A-490c, A-490d and A-490e amend it: a module the versions hold none of is filled from the highest tag of its
|
||||
// Go package in the repository it was taken in from, per module and not once; a forge that does not answer fills nothing and
|
||||
// raises call-unanswered after five minutes.
|
||||
// raises indexer-call-unanswered after five minutes.
|
||||
func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC)
|
||||
@@ -623,7 +623,7 @@ func TestAFloorEntryIsReplacedByThePublishHeardAtItsVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0668 §3, design 47 G A-490i: the finding call-unanswered of a module's floor clears, with its call-since key, on every way the
|
||||
// ADR 0668 §3, design 47 G A-490i: the finding indexer-call-unanswered of a module's floor clears, with its call-since key, on every way the
|
||||
// fill ends but the end of its context: a release gave the module a version, the module left the trunk, or the callee answered with a
|
||||
// failure (FloorModules never picks the module again to clear it).
|
||||
func TestTheFloorFindingClearsOnEveryExitOfTheFill(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// envAskUnansweredBound names the file the manifest writes the setting ask-unanswered-bound into, `${config:<key>}`
|
||||
// (design 47 H H-A-238 (1)).
|
||||
const envAskUnansweredBound = "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE"
|
||||
|
||||
// envAskSilentBound names the file of the setting ask-silent-bound (H-A-243 (2)).
|
||||
const envAskSilentBound = "MESH_RUNNER_ASK_SILENT_BOUND_FILE"
|
||||
|
||||
// askUnansweredBound reads the setting ask-unanswered-bound at the start, a Go duration (H-A-235, H-A-238 (1)): its default
|
||||
// ten minutes when the file is missing or empty, an error naming the file for a content that is no positive duration.
|
||||
func askUnansweredBound(getenv func(string) string) (time.Duration, error) {
|
||||
return durationSetting(getenv, envAskUnansweredBound, pipeline.DefaultAskUnansweredBound)
|
||||
}
|
||||
|
||||
// askSilentBound reads the setting ask-silent-bound at the start, a Go duration, forty-five minutes by default (H-A-243 (2)).
|
||||
func askSilentBound(getenv func(string) string) (time.Duration, error) {
|
||||
return durationSetting(getenv, envAskSilentBound, pipeline.DefaultAskSilentBound)
|
||||
}
|
||||
|
||||
func durationSetting(getenv func(string) string, env string, def time.Duration) (time.Duration, error) {
|
||||
v, err := settingText(getenv(env))
|
||||
if err != nil || v == "" {
|
||||
return def, err
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(env), v)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// askClock is the bound of every ask but a test run's over the state bounds (H-A-238 (5)): a key armed with its own time to
|
||||
// live before the ask is put, deleted at its holder's first word, its expiry heard by watchBounds; the finding call-unanswered,
|
||||
// scope call, id the ask, raised at the second bound and cleared when the ask is answered after all (H-A-238 (2)).
|
||||
func askClock(ctx context.Context, bounds jetstream.KeyValue, bound, silent time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock {
|
||||
return &pipeline.AskClock{Bound: bound, Silent: silent,
|
||||
Arm: func(key string, ttl time.Duration) error {
|
||||
_, err := bounds.Create(ctx, key, []byte(`{"ask":"`+key+`"}`), jetstream.KeyTTL(ttl))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return nil // armed already: the same ask put again at a start keeps its first bound
|
||||
}
|
||||
return err
|
||||
},
|
||||
// A first word is heard for every ask, most of them unbounded or answered already: a key not there is deleted of
|
||||
// nothing and writes no marker.
|
||||
Unbound: func(key string) error {
|
||||
if _, err := bounds.Get(ctx, key); errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bounds.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Raise: func(onSeat, ask string, within time.Duration) error {
|
||||
if w == nil {
|
||||
return errors.New("the runner has no finding writer")
|
||||
}
|
||||
return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: ask, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning,
|
||||
Summary: fmt.Sprintf("the ask %s of the seat %s is unanswered", ask, onSeat),
|
||||
Headline: "A step's ask is not answered",
|
||||
Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s and heard no word of its holder about it within %s, so its step ended could not run and its pipeline went on (H-A-235, H-A-243).", ask, onSeat, within),
|
||||
Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: ask}})
|
||||
},
|
||||
Clear: func(_, ask string) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
return w.Clear(ctx, mcc.ScopeCall, ask, mcc.KindCallUnanswered, "the ask is answered")
|
||||
}}
|
||||
}
|
||||
|
||||
// answered deletes the bounds of the ask on onSeat at its holder's first word, said when it fails: the bound then passes and
|
||||
// its expiry finds the step answered or asks it once more.
|
||||
func answered(c *pipeline.AskClock, onSeat, ask string, logf func(string, ...any)) {
|
||||
if err := c.Answered(onSeat, ask); err != nil {
|
||||
logf("the bounds of ask %s on %s are not deleted at its holder's word: %v", ask, onSeat, err)
|
||||
}
|
||||
}
|
||||
|
||||
// hearWantsProgress takes a step progress of the seat mesh-wants as its holder's first word on the ask it names (H-A-238 (3)).
|
||||
func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ...any)) error {
|
||||
return hub.Handle(wants.SubjectStepProgressAll, func(m jetstream.Msg) {
|
||||
var ev wants.StepProgressEvent
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
logf("%s is no step-progress: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
answered(c, wants.Seat, ev.Ask, logf)
|
||||
_ = m.Ack()
|
||||
})
|
||||
}
|
||||
|
||||
// holdersNotReadDelay is how long a running heard before the seat-holders view is read waits before it is heard again.
|
||||
const holdersNotReadDelay = 2 * time.Second
|
||||
|
||||
// hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)).
|
||||
func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error {
|
||||
return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) {
|
||||
var ev nodeengine.Running
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Module == "" {
|
||||
logf("%s is no running: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if ev.Stopped {
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
n, err := started(ev.Module)
|
||||
switch {
|
||||
case errors.Is(err, pipeline.ErrHoldersNotRead):
|
||||
_ = m.NakWithDelay(holdersNotReadDelay) // heard again once the view is read, without spinning until then
|
||||
return
|
||||
case err != nil:
|
||||
logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err)
|
||||
}
|
||||
_ = m.Ack()
|
||||
})
|
||||
}
|
||||
|
||||
// askBoundExpired takes the expiry of an ask's bound (H-A-235): the first asks once more, the second ends its step.
|
||||
func askBoundExpired(unanswered func(onSeat, ask string, again bool) error, logf func(string, ...any)) func(key string) bool {
|
||||
return func(key string) bool {
|
||||
s, ask, again, ok := pipeline.ParseAskBound(key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if err := unanswered(s, ask, again); err != nil {
|
||||
logf("the bound of ask %s on %s passed and is not taken: %v", ask, s, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -72,8 +72,8 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func
|
||||
}
|
||||
}
|
||||
|
||||
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none
|
||||
// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them).
|
||||
// boundExpired takes the expiry of a drop's key of the state bounds: it says the drop unanswered (A-222); an ask's key is
|
||||
// taken by askBoundExpired (H-A-235).
|
||||
func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
|
||||
return func(key string) {
|
||||
if s, ask, ok := pipeline.ParseDropBound(key); ok {
|
||||
|
||||
@@ -2,9 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
|
||||
@@ -50,3 +53,25 @@ func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) {
|
||||
t.Errorf("an answered drop: %v, cleared %v, unbound %v", err, w.cleared, unbound)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-238 (5), review: at a start, a first bound's expiry is taken only when the first read holds no entry for its second
|
||||
// bound (put again and taken: a delete marker; put again: a live put; both passed: an expiry, the second taken alone); a
|
||||
// second bound's expiry is taken; a delete marker or a drop's expiry is none.
|
||||
func TestMissedBoundsAtTheStart(t *testing.T) {
|
||||
const s = "node-build-go"
|
||||
key := func(a string) string { return pipeline.AskBoundKey(s, a) }
|
||||
again := func(a string) string { return pipeline.AskAgainBoundKey(s, pipeline.AgainID(a)) }
|
||||
last := map[string]jetstream.KeyValueOp{
|
||||
key("lostaaaa000001"): jetstream.KeyValuePurge, // passed while down, never asked again: taken
|
||||
key("takenaaa000002"): jetstream.KeyValuePurge, again("takenaaa000002"): jetstream.KeyValueDelete, // asked again and taken
|
||||
key("askedaaa000003"): jetstream.KeyValuePurge, again("askedaaa000003"): jetstream.KeyValuePut, // asked again, bound live
|
||||
key("bothaaaa000004"): jetstream.KeyValuePurge, again("bothaaaa000004"): jetstream.KeyValuePurge, // both passed
|
||||
key("wordaaaa000005"): jetstream.KeyValueDelete, // a first word
|
||||
pipeline.DropBoundKey(s, "dropaaaa000006"): jetstream.KeyValuePurge,
|
||||
}
|
||||
want := []string{key("lostaaaa000001"), again("bothaaaa000004")}
|
||||
slices.Sort(want)
|
||||
if got := missedBounds(last); !slices.Equal(got, want) {
|
||||
t.Errorf("missedBounds = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,8 +67,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers")
|
||||
// The bound of each superseded step's drop and of each ask: a key with its own time to live, its expiry heard (A-222, A-77;
|
||||
// H-A-238 (5)).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop or an ask no holder answers")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -160,6 +161,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
hearer.AskSteps = starter.AskSteps
|
||||
// Every ask but a test run's is bounded until its holder's first word (H-A-235, H-A-238), its bounds keys of the state
|
||||
// bounds.
|
||||
askBound, err := askUnansweredBound(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
silentBound, err := askSilentBound(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Printf("asks: an ask unanswered bound %s, asked once more, then its step ends could not run; an ask of a seat that says nothing before its end bound %s", askBound, silentBound)
|
||||
clock := askClock(ctx, bounds, askBound, silentBound, rv.Findings, cred.Module, time.Now)
|
||||
starter.Asks, hearer.Asks = clock, clock
|
||||
starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) }
|
||||
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
|
||||
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
|
||||
@@ -297,10 +311,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return hearer.RunTaken(ask, node)
|
||||
}
|
||||
if err := hearAskTaken(heard, taken, logger.Printf); err != nil {
|
||||
if err := hearAskTaken(heard, taken, clock, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
if err := hearWantsProgress(heard, clock, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
// The seat's write verbs are answered by the process that holds the pipelines (A-107); made before the bounds are heard,
|
||||
// since an ask's expired bound asks it once more through them (H-A-235).
|
||||
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
|
||||
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
|
||||
if err := hearHolderStarted(heard, verbs.HolderStarted, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf)
|
||||
expired := func(key string) {
|
||||
if !askExpired(key) {
|
||||
dropExpired(key)
|
||||
}
|
||||
}
|
||||
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -336,9 +366,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The steps' ends are kept seat events, each on its own stream (step-ended of the build seats, mesh-builds and
|
||||
// mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24).
|
||||
logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred))
|
||||
// The seat's write verbs are answered here, by the process that holds the pipelines (A-107).
|
||||
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
|
||||
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
|
||||
stopVerbs, err := answerVerbs(calls, seats, verbs)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+69
-1
@@ -107,6 +107,17 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
const router = "operator-router"
|
||||
answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{
|
||||
map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}})
|
||||
// Each mesh-builds step ask as its holder hears it, on its plain subscription.
|
||||
var buildAsksMu sync.Mutex
|
||||
buildAsks := map[string]int{}
|
||||
if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) {
|
||||
buildAsksMu.Lock()
|
||||
buildAsks[m.Header.Get(nats.MsgIdHdr)]++
|
||||
buildAsksMu.Unlock()
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buildAsked := func(id string) int { buildAsksMu.Lock(); defer buildAsksMu.Unlock(); return buildAsks[id] }
|
||||
records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -153,6 +164,8 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
if err := os.WriteFile(waitingCap, []byte("1\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
askBound := filepath.Join(dir, "ask-unanswered-bound")
|
||||
silentBound := filepath.Join(dir, "ask-silent-bound")
|
||||
logs := &syncBuffer{}
|
||||
start := func() func() {
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
@@ -164,6 +177,10 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
return broker
|
||||
case envPipelinesWaitingCap:
|
||||
return waitingCap
|
||||
case envAskUnansweredBound:
|
||||
return askBound
|
||||
case envAskSilentBound:
|
||||
return silentBound
|
||||
}
|
||||
return ""
|
||||
}, logs)
|
||||
@@ -359,9 +376,16 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
}
|
||||
|
||||
// At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears
|
||||
// pipelines-stalled; the start begins no pipeline of its own.
|
||||
// pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second, its
|
||||
// ask-silent-bound four.
|
||||
stopRunner()
|
||||
setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1)
|
||||
if err := os.WriteFile(askBound, []byte("1s\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(silentBound, []byte("4s\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := pipelineKeys(t, ctx, pipelines)
|
||||
stopRunner = start()
|
||||
outcome("main.10", seat.PipelineOutcomeRunning)
|
||||
@@ -370,6 +394,50 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
if after := pipelineKeys(t, ctx, pipelines); !slices.Equal(before, after) {
|
||||
t.Fatalf("the start wrote pipelines %v, want %v", after, before)
|
||||
}
|
||||
|
||||
// H-A-243: main.10's build ask on mesh-builds, whose holder says nothing before its end, is never asked again for
|
||||
// ask-unanswered-bound (a build running four of those bounds is not touched); its one bound,
|
||||
// ask-silent-bound, ends its step could not run "its ask <id> had no end", the pipeline ending failed, its turn freed and
|
||||
// the finding call-unanswered naming the ask.
|
||||
build := stepFor(get("main.10"), "app", seat.StepResultStepBuild)
|
||||
if build.Ask == "" || build.Started == nil {
|
||||
t.Fatalf("main.10's build = %+v, want it asked", build)
|
||||
}
|
||||
// Review of H-A-238 (5): the bound expires while the runner is down; at its next start the expiry marker it left is
|
||||
// read and taken at once, never a fresh bound's full length later.
|
||||
boundsKV, err := js.KeyValue(ctx, module+"_bounds")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
_, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask))
|
||||
return err == nil
|
||||
})
|
||||
stopRunner()
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
_, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask))
|
||||
return errors.Is(err, jetstream.ErrKeyNotFound)
|
||||
})
|
||||
restarted := time.Now()
|
||||
stopRunner = start()
|
||||
outcome("main.10", seat.PipelineOutcomeFailed)
|
||||
if e := stepFor(get("main.10"), "app", seat.StepResultStepBuild).Ended; e == nil || e.Sub(restarted) >= 3*time.Second {
|
||||
t.Fatalf("main.10's build ended %v, the runner restarted at %v; want its missed bound taken at the start", e, restarted)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "expired while the runner was away: taken now") {
|
||||
t.Fatal("the start did not say the bound it took")
|
||||
}
|
||||
s := stepFor(get("main.10"), "app", seat.StepResultStepBuild)
|
||||
if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second {
|
||||
t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s)
|
||||
}
|
||||
if strings.Contains(logs.String(), ": asked once more as ") || buildAsked(pipeline.AgainID(build.Ask)) != 0 {
|
||||
t.Fatalf("the silent build's ask was asked again for ask-unanswered-bound (put %d times, again %d)", buildAsked(build.Ask), buildAsked(pipeline.AgainID(build.Ask)))
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e, err := findings.Get(ctx, "call."+build.Ask+".call-unanswered")
|
||||
return err == nil && strings.Contains(string(e.Value()), build.Ask) && len(running()) == 0
|
||||
})
|
||||
}
|
||||
|
||||
func stepFor(p seat.Pipeline, module, step string) seat.StepResult {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -141,6 +142,8 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
logf("the step-ended of ask %s is not taken yet, heard again: %v", ev.ID, err)
|
||||
_ = m.Nak()
|
||||
default:
|
||||
// The step's end is its holder's word on the ask, the first for mesh-builds (H-A-238 (3)).
|
||||
answered(h.Asks, bs.Seat, ev.ID, logf)
|
||||
_ = m.Ack()
|
||||
}
|
||||
})
|
||||
@@ -153,8 +156,9 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
|
||||
// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of
|
||||
// that node's holder, and begins the clock of a test run's ask (§725).
|
||||
func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error {
|
||||
func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline.AskClock, logf func(string, ...any)) error {
|
||||
for _, ts := range pipeline.TakenSeats {
|
||||
ts := ts
|
||||
err := hub.Handle(ts.Subject, func(m jetstream.Msg) {
|
||||
var ev nodebuildgo.AskTaken
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
@@ -167,6 +171,8 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
// The seat's ask-taken is its holder's first word on the ask (H-A-238 (3)).
|
||||
answered(c, ts.Seat, ev.Ask, logf)
|
||||
_ = m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
@@ -176,14 +182,17 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st
|
||||
return nil
|
||||
}
|
||||
|
||||
// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered
|
||||
// unanswered (A-222, A-77): no timer in the process.
|
||||
// watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235,
|
||||
// H-A-238 (5)): no timer in the process. An ask's bound that expired while the runner was down left its marker before the
|
||||
// watch began: the watch's first entries, each key's last, are kept to their end, and the ask bounds missedBounds names are
|
||||
// taken as expired then. Then every expiry is heard as it comes.
|
||||
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
|
||||
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
w, err := bounds.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
last := map[string]jetstream.KeyValueOp{} // each key's last entry, until the first read ends
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -193,16 +202,49 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePurge {
|
||||
if e == nil {
|
||||
for _, key := range missedBounds(last) {
|
||||
logf("the bound %s expired while the runner was away: taken now", key)
|
||||
expired(key)
|
||||
}
|
||||
last = nil // the first read is over: what follows is new
|
||||
continue
|
||||
}
|
||||
expired(e.Key())
|
||||
if last != nil {
|
||||
last[e.Key()] = e.Operation()
|
||||
continue
|
||||
}
|
||||
if e.Operation() == jetstream.KeyValuePurge {
|
||||
expired(e.Key())
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// missedBounds are the ask bounds the first read found expired that are still to be taken (H-A-238 (5)): a second bound's
|
||||
// expiry always; a first bound's only when the read holds no entry at all for its second bound's key, since a live second
|
||||
// bound, its expiry or its delete (the holder's ask-taken) says the ask was asked again already. A delete marker is a first
|
||||
// word, never an expiry; a drop's bound is not taken here.
|
||||
func missedBounds(last map[string]jetstream.KeyValueOp) []string {
|
||||
var out []string
|
||||
for key, op := range last {
|
||||
if op != jetstream.KeyValuePurge {
|
||||
continue
|
||||
}
|
||||
onSeat, ask, again, ok := pipeline.ParseAskBound(key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, asked := last[pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask))]; again || !asked {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a
|
||||
// name or lies beneath it, in one pass over the bucket.
|
||||
func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) {
|
||||
|
||||
+30
-3
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
@@ -99,6 +100,19 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// mesh-builds is held by the builder, which takes its asks on a plain subscription and says nothing before an end.
|
||||
if _, err := holders.Put(ctx, builds.Seat, []byte(`{"holders":[{"module":"mesh-delivery-builder"}]}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buildAsksMu sync.Mutex
|
||||
buildAsks := map[string]int{}
|
||||
if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) {
|
||||
buildAsksMu.Lock()
|
||||
buildAsks[m.Header.Get(nats.MsgIdHdr)]++
|
||||
buildAsksMu.Unlock()
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The registrations: app tests its program and tools bundle from one directory, its unit step required and its
|
||||
// integration step optional; web is built by node-build-node.
|
||||
@@ -331,6 +345,17 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
const p3 = "main.1"
|
||||
takeIn("app", "c1", "src2:aa", 3, 1)
|
||||
placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747)
|
||||
// H-A-243 (1): the node-engine's running of the builder puts its open build ask again, under its own id: an ask put while
|
||||
// it was down is lost; a running of a module holding no silent seat puts nothing again.
|
||||
firstBuild := step(read(p3), "", seat.StepResultStepBuild).Ask
|
||||
publish(nodeengine.SubjectRunning("n1", "mesh-build-go"), nodeengine.Running{Module: "mesh-build-go", Node: "n1", At: time.Now().UTC()})
|
||||
publish(nodeengine.SubjectRunning("n1", "mesh-delivery-builder"), nodeengine.Running{Module: "mesh-delivery-builder", Node: "n1", At: time.Now().UTC()})
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
buildAsksMu.Lock()
|
||||
n := buildAsks[firstBuild]
|
||||
buildAsksMu.Unlock()
|
||||
return n == 2 && strings.Count(logs.String(), "the ask "+firstBuild+" of the build step of "+p3+" is put again on mesh-builds") == 1
|
||||
})
|
||||
passBuild(p3, "app", "c1", "src2:aa")
|
||||
unitRun, unit := runOfStep(p3, "app:program", "unit")
|
||||
integRun, integ := runOfStep(p3, "app:program", "integration")
|
||||
@@ -350,7 +375,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask {
|
||||
t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s)
|
||||
}
|
||||
// Its wait has no bound (§647 as A-466y amends it): the state bounds holds no key for a queued ask, only drops' keys.
|
||||
// A test run's ask keeps §725's bounds alone (H-A-238 (4)): the state bounds holds no key naming it, only drops' and other
|
||||
// asks' keys.
|
||||
boundsKV, err := js.KeyValue(ctx, module+"_bounds")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -359,8 +385,9 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
for _, k := range ks {
|
||||
if !strings.HasPrefix(k, pipeline.DropBoundPrefix) {
|
||||
t.Errorf("the state bounds holds %s after the asks were put; want no bound of a queued ask", k)
|
||||
_, ask, _, isAsk := pipeline.ParseAskBound(k)
|
||||
if (!isAsk && !strings.HasPrefix(k, pipeline.DropBoundPrefix)) || ask == unit.Ask || ask == integ.Ask {
|
||||
t.Errorf("the state bounds holds %s after the asks were put; want no bound of a test run's ask", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ require (
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// DefaultAskUnansweredBound is the setting ask-unanswered-bound's default (design 47 H H-A-235, H-A-238 (1)): ten minutes
|
||||
// from an ask until its holder's first word.
|
||||
const DefaultAskUnansweredBound = 10 * time.Minute
|
||||
|
||||
// DefaultAskSilentBound is the setting ask-silent-bound's default (H-A-243 (2)): forty-five minutes from an ask on a seat whose
|
||||
// holder says no word before the step's end (mesh-builds, mesh-wants) until that end.
|
||||
const DefaultAskSilentBound = 45 * time.Minute
|
||||
|
||||
// Silent says whether the holder of a seat the runner asks says nothing of an ask before its step's end: every seat but the
|
||||
// build seats, which say ask-taken (H-A-241, H-A-243).
|
||||
func Silent(onSeat string) bool { return !BoundSeat(onSeat) }
|
||||
|
||||
// AgainID is the id an ask is put again under (H-A-243 (3)): the first id's time part and six hex digits of the sha256 of the
|
||||
// first id and ".2", the form the holders accept, so no duplicate window is relied on.
|
||||
func AgainID(id string) string {
|
||||
if len(id) <= 6 {
|
||||
return id
|
||||
}
|
||||
sum := sha256.Sum256([]byte(id + ".2"))
|
||||
return id[:len(id)-6] + hex.EncodeToString(sum[:3])
|
||||
}
|
||||
|
||||
// The keys of the state bounds that time an ask the runner put (H-A-238 (5)): `ask.<seat>.<ask>` armed before the ask is put,
|
||||
// `ask2.<seat>.<ask>` armed when it is asked once more, each with the setting as its own time to live.
|
||||
const (
|
||||
AskBoundPrefix = "ask."
|
||||
AskAgainBoundPrefix = "ask2."
|
||||
)
|
||||
|
||||
// AskBoundKey is the key of the first bound of the ask on onSeat; AskAgainBoundKey that of its second.
|
||||
func AskBoundKey(onSeat, ask string) string { return AskBoundPrefix + onSeat + "." + ask }
|
||||
func AskAgainBoundKey(onSeat, ask string) string { return AskAgainBoundPrefix + onSeat + "." + ask }
|
||||
|
||||
// ParseAskBound reads a key of the state bounds as an ask's bound: the seat, the ask and whether it is the second bound.
|
||||
func ParseAskBound(key string) (onSeat, ask string, again, ok bool) {
|
||||
rest, found := strings.CutPrefix(key, AskAgainBoundPrefix)
|
||||
again = found
|
||||
if !found {
|
||||
if rest, found = strings.CutPrefix(key, AskBoundPrefix); !found {
|
||||
return "", "", false, false
|
||||
}
|
||||
}
|
||||
onSeat, ask, found = strings.Cut(rest, ".")
|
||||
return onSeat, ask, again, found && onSeat != "" && ask != "" && !strings.Contains(ask, ".")
|
||||
}
|
||||
|
||||
// UnansweredWhy is the why of a step whose ask was not answered within its two bounds (H-A-235).
|
||||
func UnansweredWhy(ask string) string { return "its ask " + ask + " was not answered" }
|
||||
|
||||
// SilentWhy is the why of a step on a silent seat whose ask had no end within ask-silent-bound (H-A-243 (2)).
|
||||
func SilentWhy(ask string) string { return "its ask " + ask + " had no end" }
|
||||
|
||||
// AskClock bounds every ask the runner puts on a seat but a test run's, which keeps §725's bounds (H-A-235, H-A-238 (4)):
|
||||
// Arm writes a key of the state bounds with Bound as its own time to live, Unbound deletes one; Raise raises the finding
|
||||
// call-unanswered of an ask that ended its step, Clear clears it once the ask is answered after all (H-A-238 (2)).
|
||||
type AskClock struct {
|
||||
// Bound is ask-unanswered-bound, for a seat that says ask-taken; Silent is ask-silent-bound, for a silent seat.
|
||||
Bound time.Duration
|
||||
Silent time.Duration
|
||||
Arm func(key string, ttl time.Duration) error
|
||||
Unbound func(key string) error
|
||||
Raise func(onSeat, ask string, bound time.Duration) error
|
||||
Clear func(onSeat, ask string) error
|
||||
}
|
||||
|
||||
// arm arms the first bound of a, before it is put (H-A-238 (5)); a test run's ask, and a nil clock, arm none.
|
||||
func (c *AskClock) arm(a Asked) error {
|
||||
if c == nil || c.Arm == nil || a.Run != "" {
|
||||
return nil
|
||||
}
|
||||
if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.boundOf(a.Seat)); err != nil {
|
||||
return fmt.Errorf("the bound of ask %s is not armed: %w", a.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// boundOf is the bound of an ask on onSeat: ask-silent-bound for a silent seat (H-A-243 (2)), else ask-unanswered-bound.
|
||||
func (c *AskClock) boundOf(onSeat string) time.Duration {
|
||||
if !Silent(onSeat) {
|
||||
return c.bound()
|
||||
}
|
||||
if c.Silent > 0 {
|
||||
return c.Silent
|
||||
}
|
||||
return DefaultAskSilentBound
|
||||
}
|
||||
|
||||
func (c *AskClock) bound() time.Duration {
|
||||
if c.Bound > 0 {
|
||||
return c.Bound
|
||||
}
|
||||
return DefaultAskUnansweredBound
|
||||
}
|
||||
|
||||
// Answered deletes the bounds of the ask heard on onSeat, its holder's first word: an ask-taken, a step progress or, for a
|
||||
// silent seat, the step's end (H-A-238 (3), H-A-241). Heard under a first id, the second bound of the ask put again under
|
||||
// AgainID goes too.
|
||||
func (c *AskClock) Answered(onSeat, ask string) error {
|
||||
if c == nil || c.Unbound == nil || ask == "" {
|
||||
return nil
|
||||
}
|
||||
return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask)),
|
||||
c.Unbound(AskAgainBoundKey(onSeat, AgainID(ask))))
|
||||
}
|
||||
|
||||
// answeredLate clears the finding of the ask of step s, which ended unanswered or with no end, its holder heard after all
|
||||
// (H-A-238 (2)).
|
||||
func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) {
|
||||
if c == nil || c.Clear == nil || s.Ask == "" || (s.Why != UnansweredWhy(s.Ask) && s.Why != SilentWhy(s.Ask)) {
|
||||
return
|
||||
}
|
||||
if err := c.Clear(s.Seat, s.Ask); err != nil {
|
||||
logf("the finding of ask %s, answered after its step ended, is not cleared: %v", s.Ask, err)
|
||||
}
|
||||
}
|
||||
|
||||
// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235, H-A-243). On a silent seat its one bound,
|
||||
// ask-silent-bound, ends the step could not run "its ask <id> had no end", never asking again (that would double a running
|
||||
// build). On a seat that says ask-taken the first bound puts the ask once more under AgainID, said in the journal, its
|
||||
// second bound armed under that id; the second ends the step could not run "its ask <id> was not answered". An ended step
|
||||
// raises the finding call-unanswered, so the pipeline ends and its turn frees. A bound whose ask no running step names any
|
||||
// longer asks nothing.
|
||||
func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error {
|
||||
h, c := v.Hearer, v.Starter.Asks
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
key, p, i, err := v.runningAsk(onSeat, ask)
|
||||
if err != nil || key == "" {
|
||||
return err
|
||||
}
|
||||
s := p.Steps[i]
|
||||
if Silent(onSeat) || again {
|
||||
why, within := UnansweredWhy(s.Ask), 2*c.bound()
|
||||
if Silent(onSeat) {
|
||||
why, within = SilentWhy(s.Ask), c.boundOf(onSeat)
|
||||
}
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: the step ends could not run (H-A-235, H-A-243)",
|
||||
s.Ask, s.Step, key, onSeat, within)
|
||||
if err := h.endUnanswered(key, s.Ask, why); err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Raise != nil {
|
||||
return c.Raise(onSeat, s.Ask, within)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
a, err := v.askAgain(p, s, AgainID(s.Ask), h.now())
|
||||
if err != nil {
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", s.Ask, s.Step, key, onSeat, c.bound(), err)
|
||||
} else {
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more as %s (H-A-235, H-A-243)", s.Ask, s.Step, key, onSeat, c.bound(), a.ID)
|
||||
}
|
||||
if c.Arm != nil {
|
||||
if aerr := c.Arm(AskAgainBoundKey(onSeat, AgainID(s.Ask)), c.bound()); aerr != nil {
|
||||
return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", s.Ask, aerr)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil // its second bound ends the step
|
||||
}
|
||||
v.Starter.carry(key, []Asked{*a})
|
||||
return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil)
|
||||
}
|
||||
|
||||
// HolderStarted puts again every open ask of each silent seat module holds (H-A-243 (1)): its holder takes its asks on a plain
|
||||
// subscription, so an ask put while it was down is lost; heard on the node-engine's running of module. Each ask is put under
|
||||
// its own id, a rerun's marks kept, said in the journal; its bound stands as armed.
|
||||
func (v Verbs) HolderStarted(module string) (int, error) {
|
||||
st := v.Starter
|
||||
if st == nil || st.Holders == nil {
|
||||
return 0, nil
|
||||
}
|
||||
var silent []string
|
||||
for onSeat := range acceptStep {
|
||||
if !Silent(onSeat) {
|
||||
continue
|
||||
}
|
||||
holders, read := st.Holders(onSeat)
|
||||
if !read {
|
||||
return 0, ErrHoldersNotRead
|
||||
}
|
||||
if containsString(holders, module) {
|
||||
silent = append(silent, onSeat)
|
||||
}
|
||||
}
|
||||
if len(silent) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
pkeys, err := allKeys(v.Hearer.Store)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
now, asked := v.Hearer.now(), 0
|
||||
var first error
|
||||
for _, key := range pkeys {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeRunning {
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var due []Asked
|
||||
for _, s := range p.Steps {
|
||||
if !open(s) || s.Ask == "" || seen[s.Ask] || !containsString(silent, s.Seat) {
|
||||
continue
|
||||
}
|
||||
seen[s.Ask] = true
|
||||
a, err := v.askAgain(p, s, s.Ask, now)
|
||||
if err != nil {
|
||||
v.Hearer.logf("the ask %s of the %s step of %s on %s is not put again as %s starts: %v", s.Ask, s.Step, key, s.Seat, module, err)
|
||||
continue
|
||||
}
|
||||
v.Hearer.logf("the ask %s of the %s step of %s is put again on %s: its holder %s started, and takes no ask put while it was down (H-A-243)",
|
||||
s.Ask, s.Step, key, s.Seat, module)
|
||||
due = append(due, *a)
|
||||
}
|
||||
if len(due) == 0 {
|
||||
continue
|
||||
}
|
||||
st.carry(key, due)
|
||||
if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
asked += len(due)
|
||||
}
|
||||
return asked, first
|
||||
}
|
||||
|
||||
// ErrHoldersNotRead is HolderStarted's answer before the seat-holders view's first view is read: the running is heard again.
|
||||
var ErrHoldersNotRead = errors.New("the seat-holders view is not read yet")
|
||||
|
||||
func containsString(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// runningAsk is the pipeline and the index of its running step asked under ask, or put again under it, of onSeat; an empty
|
||||
// key when none is.
|
||||
func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error) {
|
||||
pkeys, err := allKeys(v.Hearer.Store)
|
||||
if err != nil {
|
||||
return "", seat.Pipeline{}, -1, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if i := stepOfAsk(p, ask); i >= 0 && open(p.Steps[i]) && p.Steps[i].Seat == onSeat {
|
||||
return key, p, i, nil
|
||||
}
|
||||
}
|
||||
return "", seat.Pipeline{}, -1, nil
|
||||
}
|
||||
|
||||
// askAgain is the ask of running step s as it was put, under id: a rerun's marks kept.
|
||||
func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) {
|
||||
a, err := v.placedAsk(p, s, id, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.Rerun {
|
||||
markRerun(a, s.By)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// markRerun marks a's body a rerun by caller.
|
||||
func markRerun(a *Asked, caller string) {
|
||||
switch b := a.Ask.(type) {
|
||||
case nodebuildgo.StepAsk:
|
||||
if b.Step != nil {
|
||||
step := *b.Step
|
||||
step.Rerun, step.By = true, caller
|
||||
b.Step = &step
|
||||
}
|
||||
a.Ask = b
|
||||
case builds.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
case wants.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
}
|
||||
}
|
||||
|
||||
// endUnanswered ends the running step of pipeline key asked under ask could not run, saying why (H-A-235, H-A-243): every
|
||||
// other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when
|
||||
// nothing else is open, said and logged as any step's end.
|
||||
func (h *Hearer) endUnanswered(key, ask, why string) error {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i < 0 || !open(p.Steps[i]) {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, why, &now
|
||||
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
|
||||
var readers []int
|
||||
if s.Step == seat.StepResultStepValidate {
|
||||
readers = sameValidate(&p, i, now)
|
||||
_, notAsked := releaseCompositions(&p, now)
|
||||
readers = append(readers, notAsked...)
|
||||
}
|
||||
asked, err := askApproval(key, &p, now, h.Asker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ended := h.endPipeline(&p, now)
|
||||
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("pipeline %s cannot be written: %w", key, err)
|
||||
}
|
||||
h.Failures.StepEnded(h.Store, p, *s)
|
||||
for _, j := range append([]int{i}, readers...) {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
|
||||
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
|
||||
}
|
||||
}
|
||||
h.appendUnlogged(key)
|
||||
return h.said(p, asked, ended)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// H-A-235, H-A-238: every ask but a test run's is bounded from its put until its holder's first word. A bound that passes asks
|
||||
// once more under the ask's own id, its second bound armed; past the second the step ends could not run "its ask <id> was not
|
||||
// answered", the finding raised, and the pipeline ends; a first word deletes both bounds, and a late one clears the finding.
|
||||
func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) {
|
||||
v, store, said, asks, _, _ := verbsFixture()
|
||||
var armed, unbound, raised, cleared []string
|
||||
logs := []string{}
|
||||
v.Hearer.Logf = func(f string, a ...any) { logs = append(logs, f) }
|
||||
c := &AskClock{Bound: 10 * time.Minute,
|
||||
Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil },
|
||||
Unbound: func(key string) error { unbound = append(unbound, key); return nil },
|
||||
Raise: func(s, ask string, _ time.Duration) error { raised = append(raised, s+" "+ask); return nil },
|
||||
Clear: func(s, ask string) error { cleared = append(cleared, s+" "+ask); return nil }}
|
||||
v.Starter.Asks, v.Hearer.Asks = c, c
|
||||
|
||||
// A test run's ask keeps §725's bounds; a nil clock arms none.
|
||||
if err := c.arm(Asked{Seat: nodebuildgo.Seat, ID: "r1", Run: "run"}); err != nil || len(armed) != 0 {
|
||||
t.Fatalf("a test run's ask armed %v (%v)", armed, err)
|
||||
}
|
||||
if err := (*AskClock)(nil).arm(Asked{Seat: nodebuildgo.Seat, ID: "x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Every step but app:program's lint has ended; that lint failed, and its rerun's ask is armed before it is put.
|
||||
p, rev, _ := store.Get(pkey)
|
||||
for i := range p.Steps {
|
||||
if open(p.Steps[i]) {
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomePassed
|
||||
}
|
||||
}
|
||||
p.Steps[stepIndex(p, "lint", "app:program")].Outcome = seat.StepResultOutcomeFailed
|
||||
store.Update(pkey, p, rev)
|
||||
r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask := r.Result.Ask
|
||||
if want := []string{AskBoundKey(nodebuildgo.Seat, ask) + " 10m0s"}; !slices.Equal(armed, want) || len(asks.ids) != 1 {
|
||||
t.Fatalf("armed %v asking %v, want %v before one ask", armed, asks.ids, want)
|
||||
}
|
||||
// The first bound passes: asked once more under the id derived from the first, in the form the holders accept, a rerun
|
||||
// still, the second bound armed under it and said (H-A-243 (3)).
|
||||
again := AgainID(ask)
|
||||
if err := sdkstep.CheckAskID(again); err != nil || again == ask || again[:len(again)-6] != ask[:len(ask)-6] {
|
||||
t.Fatalf("AgainID(%s) = %s (%v), want the first's time part and other hex digits in the ask id form", ask, again, err)
|
||||
}
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asks.ids) != 2 || asks.ids[1] != again || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, again)+" 10m0s" {
|
||||
t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, again)
|
||||
}
|
||||
if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != again || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" {
|
||||
t.Errorf("the ask put again = %+v", asks.asks[1])
|
||||
}
|
||||
if !slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "asked once more") }) {
|
||||
t.Errorf("no journal line names the ask asked again: %v", logs)
|
||||
}
|
||||
if got, _, _ := store.Get(pkey); got.Steps[stepIndex(got, "lint", "app:program")].Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Fatal("the step ended at the first bound")
|
||||
}
|
||||
// The second bound passes: the step ends could not run, the finding is raised, the pipeline ends failed.
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, again, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
s := got.Steps[stepIndex(got, "lint", "app:program")]
|
||||
if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+ask+" was not answered" || s.Ended == nil {
|
||||
t.Errorf("the step = %+v", s)
|
||||
}
|
||||
if got.Outcome != seat.PipelineOutcomeFailed || len(asks.ids) != 2 {
|
||||
t.Errorf("the pipeline is %s (%s), asked %d, want failed and nothing more asked", got.Outcome, got.Why, len(asks.ids))
|
||||
}
|
||||
if !slices.Equal(raised, []string{nodebuildgo.Seat + " " + ask}) || !slices.Contains(*said, "pipeline-ended.main.5") {
|
||||
t.Errorf("raised %v, said %v", raised, *said)
|
||||
}
|
||||
// A bound whose step ended asks nothing; a first word deletes both bounds; a late end clears the finding.
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil || len(asks.ids) != 2 {
|
||||
t.Errorf("a bound of an ended step asked %v (%v)", asks.ids, err)
|
||||
}
|
||||
if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, again)}) {
|
||||
t.Errorf("a first word deleted %v (%v)", unbound, err)
|
||||
}
|
||||
if err := v.Hearer.Taken(again, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) {
|
||||
t.Errorf("a late ask-taken cleared %v (%v)", cleared, err)
|
||||
}
|
||||
for _, k := range []string{AskBoundKey("mesh-wants", "a1"), AskAgainBoundKey("mesh-wants", "a1")} {
|
||||
if s, a, again, ok := ParseAskBound(k); !ok || s != "mesh-wants" || a != "a1" || again != strings.HasPrefix(k, AskAgainBoundPrefix) {
|
||||
t.Errorf("ParseAskBound(%s) = %s %s %v %v", k, s, a, again, ok)
|
||||
}
|
||||
}
|
||||
if _, _, _, ok := ParseAskBound(DropBoundKey("node-build-go", "a1")); ok {
|
||||
t.Error("a drop's bound read as an ask's")
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-243: an ask on a seat whose holder says nothing before the step's end (mesh-builds) is never asked again for a bound:
|
||||
// its one bound, ask-silent-bound, ends the step could not run "its ask <id> had no end", the finding raised. The node-engine's
|
||||
// running of the seat's holder puts every open ask of the seat again under its own id, a module holding no such seat none.
|
||||
func TestASilentSeatsAskIsPutAgainWhenItsHolderStartsAndEndsAtItsSilentBound(t *testing.T) {
|
||||
v, store, _, asks, _, _ := verbsFixture()
|
||||
var armed, raised []string
|
||||
c := &AskClock{Bound: time.Minute, Silent: 45 * time.Minute,
|
||||
Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil },
|
||||
Raise: func(s, ask string, within time.Duration) error {
|
||||
raised = append(raised, s+" "+ask+" "+within.String())
|
||||
return nil
|
||||
}}
|
||||
v.Starter.Asks, v.Hearer.Asks = c, c
|
||||
v.Starter.Holders = func(s string) ([]string, bool) {
|
||||
return map[string][]string{builds.Seat: {"mesh-delivery-builder"}}[s], true
|
||||
}
|
||||
p, rev, _ := store.Get(pkey)
|
||||
// Every step has ended but app's build, asked of mesh-builds.
|
||||
b := stepIndex(p, seat.StepResultStepBuild, "")
|
||||
for i := range p.Steps {
|
||||
if open(p.Steps[i]) && i != b {
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomePassed
|
||||
}
|
||||
}
|
||||
if p.Steps[b].Seat != builds.Seat || !open(p.Steps[b]) {
|
||||
t.Fatalf("the build step = %+v, want running on mesh-builds", p.Steps[b])
|
||||
}
|
||||
store.Update(pkey, p, rev)
|
||||
build := p.Steps[b].Ask
|
||||
if err := c.arm(Asked{Seat: builds.Seat, ID: build}); err != nil || !slices.Equal(armed, []string{AskBoundKey(builds.Seat, build) + " 45m0s"}) {
|
||||
t.Fatalf("armed %v (%v), want the silent bound", armed, err)
|
||||
}
|
||||
// A module holding no silent seat puts nothing again; the builder's running puts the build again under its own id.
|
||||
if n, err := v.HolderStarted("mesh-build-go"); err != nil || n != 0 || len(asks.ids) != 0 {
|
||||
t.Fatalf("another holder's start put %d (%v)", n, err)
|
||||
}
|
||||
if n, err := v.HolderStarted("mesh-delivery-builder"); err != nil || n != 1 || len(asks.ids) != 1 || asks.ids[0] != build {
|
||||
t.Fatalf("the builder's start put %d asks %v (%v), want %s again", n, asks.ids, err, build)
|
||||
}
|
||||
if _, ok := asks.asks[0].(builds.StepRun); !ok || asks.at[0] != builds.AcceptSubjectStep {
|
||||
t.Errorf("the ask put again = %+v on %s", asks.asks[0], asks.at[0])
|
||||
}
|
||||
// The silent bound passes: no ask again, the step ends could not run, the finding is raised, the pipeline ends.
|
||||
if err := v.AskUnanswered(builds.Seat, build, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
s := got.Steps[b]
|
||||
if len(asks.ids) != 1 || s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build+" had no end" || got.Outcome != seat.PipelineOutcomeFailed {
|
||||
t.Errorf("asked %v; the build %+v; the pipeline %s", asks.ids, s, got.Outcome)
|
||||
}
|
||||
if !slices.Equal(raised, []string{builds.Seat + " " + build + " 45m0s"}) {
|
||||
t.Errorf("raised %v", raised)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-252: the pipeline's one validate ask names no mover (H-A-51), and its holder's end names the mover whose body it took:
|
||||
// it is matched by its ask alone and ends the validate step of every mover under it, whichever mover it names.
|
||||
func TestAValidateEndNamingOneMoverEndsEveryMoversValidate(t *testing.T) {
|
||||
h, store, _, _, _, _ := fixture()
|
||||
p, rev, _ := store.Get(pkey)
|
||||
p.Movers = []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "lib", Commit: "c2", Fingerprint: "src2:bb"}}
|
||||
p.Steps = []seat.StepResult{
|
||||
{Module: "app", Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "v1", Seat: wants.Seat},
|
||||
{Module: "lib", Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "v1", Seat: wants.Seat}}
|
||||
store.Update(pkey, p, rev)
|
||||
var bs StepSeat
|
||||
for _, x := range StepSeats {
|
||||
if x.Seat == wants.Seat {
|
||||
bs = x
|
||||
}
|
||||
}
|
||||
ev := StepEnded{ID: "v1", Node: "n1", Pipeline: pkey, Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Step: seat.StepResultStepValidate, Outcome: "passed"}
|
||||
if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
for _, s := range got.Steps {
|
||||
if s.Outcome != seat.StepResultOutcomePassed {
|
||||
t.Errorf("%s's validate = %s, want passed by the one end", s.Module, s.Outcome)
|
||||
}
|
||||
}
|
||||
if got.Outcome != seat.PipelineOutcomePassed {
|
||||
t.Errorf("the pipeline = %s (%s), want passed", got.Outcome, got.Why)
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
)
|
||||
|
||||
// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes,
|
||||
// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as
|
||||
// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725).
|
||||
// a key of the state `bounds` with its own time to live. An ask waiting for its holder is bounded by ask-unanswered-bound
|
||||
// (H-A-235, H-A-238; AskClock), a test run's by run-untaken-bound (§725).
|
||||
const DropBound = 5 * time.Minute
|
||||
|
||||
// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it.
|
||||
|
||||
@@ -387,17 +387,23 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar
|
||||
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}}
|
||||
}
|
||||
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
|
||||
// holder with no bound of the runner's own (§535, §647 as A-466y amends them).
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool) error {
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. arm, when not nil, arms
|
||||
// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put.
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error) error {
|
||||
var first error
|
||||
for _, a := range asks {
|
||||
accept, ok := acceptStep[a.Seat]
|
||||
if to, named := acceptRecovery[a.Seat]; named && recovery != nil && recovery(a.Commit) {
|
||||
accept = to
|
||||
}
|
||||
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
|
||||
if ok {
|
||||
var err error
|
||||
switch {
|
||||
case !ok:
|
||||
err = fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
|
||||
case arm != nil:
|
||||
err = arm(a)
|
||||
}
|
||||
if err == nil {
|
||||
err = asker.AskStep(accept, a.ID, a.Ask)
|
||||
}
|
||||
if err != nil && first == nil {
|
||||
|
||||
@@ -176,6 +176,52 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-244 (1), H-A-223: a mover a newer revision moves leaves an older waiting pipeline also when the newer one is placed at
|
||||
// once, not only when it waits: the waiting pipeline keeps the movers still its own, and with none left ends superseded,
|
||||
// never placed.
|
||||
func TestANewerPlacedPipelineSupersedesAnOlderWaitingOne(t *testing.T) {
|
||||
s, mem, q, said, _, _, view, _ := queueFixture()
|
||||
take := takeIn(t, s, view)
|
||||
take(4, "app")
|
||||
take(5, "lib", "tool")
|
||||
if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeWaiting {
|
||||
t.Fatalf("main.5 = %s, want waiting", p.Outcome)
|
||||
}
|
||||
s.Queue.AtOnce = 3
|
||||
take(6, "lib")
|
||||
if p, _, _ := mem.Get("main.6"); p.Outcome != seat.PipelineOutcomeRunning {
|
||||
t.Fatalf("main.6 = %s, want placed at once", p.Outcome)
|
||||
}
|
||||
p5, _, _ := mem.Get("main.5")
|
||||
if p5.Outcome != seat.PipelineOutcomeWaiting || len(p5.Movers) != 1 || p5.Movers[0].Module != "tool" || !slices.Equal(p5.Superseded, []string{"lib"}) {
|
||||
t.Fatalf("main.5 = %s moving %+v, superseded %v; want waiting with tool alone, lib superseded", p5.Outcome, p5.Movers, p5.Superseded)
|
||||
}
|
||||
// At its turn it is placed with tool alone.
|
||||
p4, rev, _ := mem.Get("main.4")
|
||||
p4.Outcome = seat.PipelineOutcomePassed
|
||||
mem.Update("main.4", p4, rev)
|
||||
if _, err := s.Turn("main.4", view); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeRunning || len(stepsOf(p, "lib")) != 0 || len(stepsOf(p, "tool")) == 0 {
|
||||
t.Fatalf("main.5 at its turn = %s with steps %+v, want running with tool's steps alone", p.Outcome, p.Steps)
|
||||
}
|
||||
|
||||
// A waiting pipeline whose one mover a newer placed pipeline moves ends superseded, never placed.
|
||||
s, mem, q, said, _, _, view, _ = queueFixture()
|
||||
take = takeIn(t, s, view)
|
||||
take(4, "app")
|
||||
take(5, "tool")
|
||||
s.Queue.AtOnce = 3
|
||||
take(7, "tool")
|
||||
if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeSuperseded || len(p.Steps) != 0 || p.Ended == nil {
|
||||
t.Errorf("main.5 = %s with %d steps, want superseded and never placed", p.Outcome, len(p.Steps))
|
||||
}
|
||||
if _, ok := q["waiting.00000000000000000001"]; ok || !slices.Contains(*said, "pipeline-ended.main.5") {
|
||||
t.Errorf("the queue %v, said %v; want main.5's waiting key gone and its pipeline-ended said", q, *said)
|
||||
}
|
||||
}
|
||||
|
||||
// R13/AC1 (design 47 H §748, ADR 0682 §7): a newer waiting pipeline of the same branch takes the movers it also moves out of
|
||||
// an older waiting one, which ends superseded, saying wait-ended and pipeline-ended, once no mover is left; a running
|
||||
// pipeline moving the same module runs to its end.
|
||||
|
||||
@@ -183,6 +183,8 @@ type Starter struct {
|
||||
// Queue bounds the pipelines asked at once to pipelines-at-once, the others waiting in order (design 47 H §748); nil
|
||||
// places every start at once.
|
||||
Queue *Queue
|
||||
// Asks bounds every ask but a test run's until its holder's first word (H-A-235, H-A-238); nil bounds none.
|
||||
Asks *AskClock
|
||||
// RunTurn places a run on request whose turn in the queue came, from its candidate's newest revision (A-222); nil places
|
||||
// it at the runner's next start.
|
||||
RunTurn func(UnplacedRun)
|
||||
@@ -252,7 +254,7 @@ func (s *Starter) askSteps(key string, asks []Asked) error {
|
||||
var err error
|
||||
if len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) })
|
||||
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm)
|
||||
}
|
||||
s.sayStarted(key, placed)
|
||||
return err
|
||||
@@ -545,11 +547,12 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
if s.Supersede != nil {
|
||||
s.Supersede(p)
|
||||
}
|
||||
// ... and every older waiting one, whether this one waits or is placed at once (H-A-244 (1)).
|
||||
if _, err := s.supersedeWaiting(p); err != nil {
|
||||
s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err)
|
||||
}
|
||||
if !place {
|
||||
s.sayQueueWait(seat.EventWaitHeld, p, now, "")
|
||||
if _, err := s.supersedeWaiting(p); err != nil {
|
||||
s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err)
|
||||
}
|
||||
s.Queue.overCap(key, s.logf)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -176,6 +176,9 @@ type Hearer struct {
|
||||
// Unplaced says why a mover of a pipeline has no step placed, for the why of its end (H-A-171; Starter.Unplaced): it
|
||||
// waits on its registration or on a publish; nil says only that it has no step placed.
|
||||
Unplaced func(p seat.Pipeline, module string) string
|
||||
// Asks clears the finding of an ask that ended its step unanswered once it is answered after all (H-A-238 (2)); nil
|
||||
// clears none.
|
||||
Asks *AskClock
|
||||
// Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin);
|
||||
// nil keys none.
|
||||
Rejoin func(key string)
|
||||
@@ -236,6 +239,12 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
|
||||
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
|
||||
}
|
||||
i := stepOfAsk(p, ev.ID)
|
||||
// The validate ask is the pipeline's one, naming no mover (H-A-51): its end, naming the mover whose body the holder
|
||||
// took, is matched by its ask alone and ends every mover's validate step under it (H-A-252).
|
||||
oneValidate := i >= 0 && p.Steps[i].Step == seat.StepResultStepValidate && p.Steps[i].Seat == wants.Seat
|
||||
if oneValidate {
|
||||
i = validateStepOfAsk(p, ev.ID, ev.Module)
|
||||
}
|
||||
var m seat.Mover
|
||||
if i >= 0 {
|
||||
m, _ = MoverOf(p, p.Steps[i].Module)
|
||||
@@ -245,10 +254,11 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
|
||||
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
|
||||
case p.Steps[i].Seat != bs.Seat:
|
||||
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
|
||||
case ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module || ev.Step != p.Steps[i].Step:
|
||||
case ev.Step != p.Steps[i].Step || (!oneValidate && (ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module)):
|
||||
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
|
||||
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint)
|
||||
case !open(p.Steps[i]):
|
||||
h.Asks.answeredLate(p.Steps[i], h.logf)
|
||||
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
|
||||
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
|
||||
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
|
||||
@@ -583,9 +593,36 @@ func endOutcome(o string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// validateStepOfAsk is the validate step the end of the shared validate ask id ends first (H-A-252): the open one of the mover
|
||||
// the end names, else any open one under the ask (sameValidate ends the rest), else the first under it.
|
||||
func validateStepOfAsk(p seat.Pipeline, id, module string) int {
|
||||
first, anyOpen := -1, -1
|
||||
for i, s := range p.Steps {
|
||||
if s.Step != seat.StepResultStepValidate || s.Ask == "" || (s.Ask != id && AgainID(s.Ask) != id) {
|
||||
continue
|
||||
}
|
||||
if first < 0 {
|
||||
first = i
|
||||
}
|
||||
if open(s) {
|
||||
if s.Module == module {
|
||||
return i
|
||||
}
|
||||
if anyOpen < 0 {
|
||||
anyOpen = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if anyOpen >= 0 {
|
||||
return anyOpen
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func stepOfAsk(p seat.Pipeline, id string) int {
|
||||
for i, s := range p.Steps {
|
||||
if id != "" && s.Ask == id {
|
||||
// An ask put again is put under AgainID of the step's ask (H-A-243 (3)): its holder's answer under either is the step's.
|
||||
if id != "" && s.Ask != "" && (s.Ask == id || AgainID(s.Ask) == id) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
@@ -363,7 +363,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
|
||||
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
|
||||
var to []string
|
||||
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
|
||||
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }); err != nil || strings.Join(to, ",") != c.want {
|
||||
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil); err != nil || strings.Join(to, ",") != c.want {
|
||||
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,9 @@ func (h *Hearer) Taken(ask, node string) error {
|
||||
break
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i >= 0 && !open(p.Steps[i]) {
|
||||
h.Asks.answeredLate(p.Steps[i], h.logf)
|
||||
}
|
||||
if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node {
|
||||
break
|
||||
}
|
||||
@@ -163,10 +166,11 @@ func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool {
|
||||
|
||||
// supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H
|
||||
// §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its
|
||||
// pipeline-ended. newer is a pipeline just written waiting; a running pipeline's movers leave it by SupersedeRunning
|
||||
// (H-A-223). It answers the pipelines it ended.
|
||||
// pipeline-ended. newer is a pipeline just written, waiting or placed: a mover a newer revision moves leaves every pipeline not
|
||||
// ended, a waiting one as a running one (H-A-223, H-A-244 (1)), so a waiting pipeline is placed with only the movers still its
|
||||
// own; a running pipeline's movers leave it by SupersedeRunning. It answers the pipelines it ended.
|
||||
func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) {
|
||||
if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting {
|
||||
if s.Queue == nil || (newer.Outcome != seat.PipelineOutcomeWaiting && newer.Outcome != seat.PipelineOutcomeRunning) {
|
||||
return nil, nil
|
||||
}
|
||||
wkeys, pipelines, err := s.Queue.Waiting()
|
||||
@@ -193,7 +197,7 @@ func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) {
|
||||
func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, error) {
|
||||
for {
|
||||
p, rev, err := s.Store.Get(key)
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || p.Seq >= newer.Seq {
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || !olderWaiting(p, newer) {
|
||||
return false, nil
|
||||
}
|
||||
left := slices.DeleteFunc(slices.Clone(p.Movers), func(m seat.Mover) bool { return Moves(newer, m.Module) })
|
||||
@@ -232,6 +236,15 @@ func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, err
|
||||
}
|
||||
}
|
||||
|
||||
// olderWaiting says whether the waiting pipeline p is older than newer: by its sequence in the queue when newer waits too,
|
||||
// by its catalog revision when newer was placed at once (H-A-244 (1)); a run on request's pipelines are never compared.
|
||||
func olderWaiting(p, newer seat.Pipeline) bool {
|
||||
if newer.Outcome == seat.PipelineOutcomeWaiting {
|
||||
return p.Seq < newer.Seq
|
||||
}
|
||||
return p.Run == "" && newer.Run == "" && p.Revision < newer.Revision
|
||||
}
|
||||
|
||||
// supersededWhy writes, into mover module's build steps, that a dependency it builds on was taken out of p by a newer
|
||||
// pipeline, so it builds against the newest published package (design 47 H §748; H-A-53, H-A-107).
|
||||
func supersededWhy(p seat.Pipeline, module string, entries map[string]catalog.CatalogEntry, steps []seat.StepResult) {
|
||||
|
||||
@@ -3,10 +3,10 @@ package pipeline
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
@@ -297,11 +297,15 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
return -1, err
|
||||
}
|
||||
}
|
||||
oldAsk := s.Ask
|
||||
*s = seat.StepResult{Step: s.Step, Module: s.Module, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
|
||||
Outcome: seat.StepResultOutcomeWaiting, By: caller, Rerun: true}
|
||||
if ask != nil {
|
||||
s.Outcome, s.Started, s.Ask = seat.StepResultOutcomeRunning, &now, ask.ID
|
||||
}
|
||||
if ask != nil && s.Step == seat.StepResultStepValidate && s.Seat == wants.Seat {
|
||||
rerunValidate(p, i, oldAsk, caller, now)
|
||||
}
|
||||
// The pipeline runs again; an approval never asked waits again, to be asked when the step passes.
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
|
||||
@@ -321,7 +325,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
}
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -468,7 +472,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
break
|
||||
}
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -480,6 +484,29 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil
|
||||
}
|
||||
|
||||
// rerunValidate puts, with the rerun of validate step i, every other mover's validate step under the same ask back to running
|
||||
// under its one new id, a mover that left the pipeline skipped, so the pipeline keeps one validate ask (H-A-51); and each
|
||||
// composition its validate held ended not asked waits again, asked once validate passed as at the placing (§745).
|
||||
func rerunValidate(p *seat.Pipeline, i int, oldAsk, caller string, now time.Time) {
|
||||
id := p.Steps[i].Ask
|
||||
for j := range p.Steps {
|
||||
st := &p.Steps[j]
|
||||
if slices.Contains(p.Superseded, st.Module) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case j != i && st.Step == seat.StepResultStepValidate && st.Seat == wants.Seat && oldAsk != "" &&
|
||||
(st.Ask == oldAsk || st.Ask == AgainID(oldAsk)) && st.Outcome != seat.StepResultOutcomeSuperseded && st.Outcome != seat.StepResultOutcomeCancelled:
|
||||
*st = seat.StepResult{Step: st.Step, Module: st.Module, Artifact: st.Artifact, Status: st.Status, Fingerprint: st.Fingerprint, Seat: st.Seat,
|
||||
Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: id, By: caller, Rerun: true}
|
||||
case st.Step == seat.StepResultStepComposition && st.Seat == wants.Seat && st.Outcome == seat.StepResultOutcomeNotAsked &&
|
||||
strings.HasPrefix(st.Why, "validate ended "):
|
||||
*st = seat.StepResult{Step: st.Step, Module: st.Module, Artifact: st.Artifact, Status: st.Status, Fingerprint: st.Fingerprint, Seat: st.Seat,
|
||||
Outcome: seat.StepResultOutcomeWaiting, Why: "asked once validate passed"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run.
|
||||
func (v Verbs) passedReader(reader RunReader, run string) bool {
|
||||
p, _, err := v.Hearer.Store.Get(reader.Pipeline)
|
||||
@@ -500,8 +527,24 @@ func (v Verbs) passedReader(reader RunReader, run string) bool {
|
||||
}
|
||||
|
||||
// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice
|
||||
// under one id is kept once), saying it is a rerun and by whom.
|
||||
// under one id is kept once), saying it is a rerun and by whom. The validate ask is the pipeline's one, naming no mover
|
||||
// (H-A-51): its rerun asks that one ask again; a composition's rerun builds the mover's composition ask (H-A-236).
|
||||
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
|
||||
id := AskID(now, p.Pipeline, s.Module, s.Artifact, s.Step)
|
||||
if s.Seat == wants.Seat && s.Step == seat.StepResultStepValidate {
|
||||
id = AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate)
|
||||
}
|
||||
a, err := v.placedAsk(p, s, id, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
markRerun(a, caller)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// placedAsk is the ask of step s under id: a validate or composition step of the seat mesh-wants built directly for its
|
||||
// mover (H-A-236), any other the one Place makes for it.
|
||||
func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) {
|
||||
if v.Starter == nil || v.Starter.Registration == nil || v.Starter.Steps == nil || (v.View == nil && p.Run == "") {
|
||||
return nil, errors.New("the runner cannot ask a step: it holds no registration or no ask")
|
||||
}
|
||||
@@ -509,6 +552,16 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s moves no module %q", p.Pipeline, s.Module)
|
||||
}
|
||||
if s.Seat == wants.Seat {
|
||||
switch s.Step {
|
||||
case seat.StepResultStepValidate:
|
||||
a := wantsAsk(p, m, id, wants.StepRunStepValidate)
|
||||
return &a, nil
|
||||
case seat.StepResultStepComposition:
|
||||
a := wantsAsk(p, m, id, wants.StepRunStepComposition)
|
||||
return &a, nil
|
||||
}
|
||||
}
|
||||
reg, ok := v.Starter.Registration(m.Module)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s has no registration", m.Module)
|
||||
@@ -533,21 +586,9 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
|
||||
if a.ID != base {
|
||||
continue
|
||||
}
|
||||
a.ID = AskID(now, p.Pipeline, m.Module, s.Artifact, s.Step)
|
||||
switch b := a.Ask.(type) {
|
||||
case nodebuildgo.StepAsk:
|
||||
b.ID = a.ID
|
||||
if b.Step != nil {
|
||||
step := *b.Step
|
||||
step.Rerun, step.By = true, caller
|
||||
b.Step = &step
|
||||
}
|
||||
a.Ask = b
|
||||
case builds.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
case wants.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.ID = id
|
||||
if b, ok := a.Ask.(nodebuildgo.StepAsk); ok {
|
||||
b.ID = id
|
||||
a.Ask = b
|
||||
}
|
||||
return &a, nil
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
@@ -252,6 +253,47 @@ func TestRerunOfARunAsksWithItsCandidatesHalf(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-236: a rerun of a failed pipeline's validate step asks the pipeline's one validate ask again (H-A-51), naming no
|
||||
// mover in its id; a rerun of a composition step builds the mover's composition ask directly. Neither is refused "placed no
|
||||
// ask now".
|
||||
func TestRerunOfValidateOrCompositionAsksTheSharedAsk(t *testing.T) {
|
||||
v, store, _, asks, _, _ := verbsFixture()
|
||||
reg, entries := placeFixture()
|
||||
reg.Policy.Steps.Validate, reg.Policy.Steps.Composition = "required", "required"
|
||||
v.Starter.Registration = func(string) (catalog.Registration, bool) { return reg, true }
|
||||
p, rev, _ := store.Get(pkey)
|
||||
p.Steps, _ = Place(p, p.Movers[0], reg, entries, time.Unix(1, 0), "the-runner")
|
||||
for i := range p.Steps {
|
||||
switch p.Steps[i].Step {
|
||||
case seat.StepResultStepValidate:
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomeFailed
|
||||
case seat.StepResultStepComposition:
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomeNotAsked
|
||||
}
|
||||
}
|
||||
p.Outcome = seat.PipelineOutcomeFailed
|
||||
store.Update(pkey, p, rev)
|
||||
now := v.now()
|
||||
for _, c := range []struct{ step, module string }{{seat.StepResultStepValidate, ""}, {seat.StepResultStepComposition, "app"}} {
|
||||
n := len(asks.asks)
|
||||
r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Module: "app", Step: c.step, Why: "the holder restarted"})
|
||||
want := AskID(now, pkey, c.module, "", c.step)
|
||||
if err != nil || r.Result == nil || r.Result.Ask != want || r.Result.Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Fatalf("a rerun of %s = %+v, %v, want running under %s", c.step, r.Result, err, want)
|
||||
}
|
||||
if len(asks.asks) != n+1 || asks.ids[n] != want {
|
||||
t.Fatalf("a rerun of %s asked %v, want one ask under %s", c.step, asks.ids[n:], want)
|
||||
}
|
||||
b, ok := asks.asks[n].(wants.StepRun)
|
||||
if !ok || b.Step != c.step || b.Module != "app" || b.Pipeline != pkey || !b.Rerun || b.By != "operator" || b.Asker != seat.Seat {
|
||||
t.Errorf("the %s ask = %+v", c.step, asks.asks[n])
|
||||
}
|
||||
p, rev, _ = store.Get(pkey)
|
||||
p.Steps[stepIndex(p, c.step, "")].Outcome = seat.StepResultOutcomeFailed
|
||||
store.Update(pkey, p, rev)
|
||||
}
|
||||
}
|
||||
|
||||
func stepIndex(p seat.Pipeline, step, artifact string) int {
|
||||
for i, s := range p.Steps {
|
||||
if s.Step == step && s.Artifact == artifact {
|
||||
@@ -340,3 +382,44 @@ func TestStartAndAskAgain(t *testing.T) {
|
||||
t.Error("an approval running was asked again")
|
||||
}
|
||||
}
|
||||
|
||||
// Review of H-A-236 (H-A-51): a rerun of validate puts every mover's validate step under the old ask back to running under
|
||||
// the one new id, a mover that left skipped, and the compositions its failed validate ended not asked wait again; the new
|
||||
// ask's end ends both validates and asks both compositions.
|
||||
func TestAValidateRerunMovesEveryMoversValidateAndReopensItsCompositions(t *testing.T) {
|
||||
v, store, _, asks, _, _ := verbsFixture()
|
||||
p, rev, _ := store.Get(pkey)
|
||||
p.Outcome = seat.PipelineOutcomeFailed
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: "lib", Commit: "c2", Fingerprint: "src2:bb"}, seat.Mover{Module: "gone", Commit: "c3", Fingerprint: "src2:cc"})
|
||||
p.Superseded = []string{"gone"}
|
||||
notAsked := "validate ended failed: nothing is composed"
|
||||
p.Steps = nil
|
||||
for _, m := range []string{"app", "lib", "gone"} {
|
||||
p.Steps = append(p.Steps,
|
||||
seat.StepResult{Module: m, Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Seat: wants.Seat, Ask: "old", Outcome: seat.StepResultOutcomeFailed},
|
||||
seat.StepResult{Module: m, Step: seat.StepResultStepComposition, Status: seat.StepResultStatusRequired, Seat: wants.Seat, Outcome: seat.StepResultOutcomeNotAsked, Why: notAsked})
|
||||
}
|
||||
p.Steps[4].Outcome, p.Steps[5].Outcome = seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeSuperseded
|
||||
store.Update(pkey, p, rev)
|
||||
r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Module: "app", Step: seat.StepResultStepValidate, Why: "again"})
|
||||
if err != nil || len(asks.ids) != 1 || r.Result.Ask != asks.ids[0] {
|
||||
t.Fatalf("the rerun = %+v (%v), asked %v; want one validate ask", r.Result, err, asks.ids)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
for i, want := range []string{"running " + r.Result.Ask, "waiting ", "running " + r.Result.Ask, "waiting ", "superseded old", "superseded "} {
|
||||
if s := got.Steps[i]; s.Outcome+" "+s.Ask != want {
|
||||
t.Errorf("%s's %s = %s %q, want %s", s.Module, s.Step, s.Outcome, s.Ask, want)
|
||||
}
|
||||
}
|
||||
bs := StepSeats[len(StepSeats)-1]
|
||||
ev := StepEnded{ID: r.Result.Ask, Node: "n1", Pipeline: pkey, Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Step: seat.StepResultStepValidate, Outcome: "passed"}
|
||||
if _, err := v.Hearer.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ = store.Get(pkey)
|
||||
for _, i := range []int{1, 3} {
|
||||
if s := got.Steps[i]; s.Outcome != seat.StepResultOutcomeRunning || s.Ask == "" {
|
||||
t.Errorf("%s's composition = %s %q, want asked once validate passed", s.Module, s.Outcome, s.Ask)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,7 +84,7 @@ func TestManifest(t *testing.T) {
|
||||
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
|
||||
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken",
|
||||
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "node-engine.running", "mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision", "mesh-catalog.version-released", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
|
||||
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
|
||||
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed",
|
||||
@@ -199,7 +199,7 @@ func runSettingsOf(raw []byte) bool {
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return false
|
||||
}
|
||||
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
|
||||
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
|
||||
"pipelines-at-once": "number", "pipelines-waiting-cap": "number"}
|
||||
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
|
||||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
|
||||
|
||||
@@ -58,6 +58,8 @@
|
||||
"node-build-node.step-progress",
|
||||
"node-build-image.step-progress",
|
||||
"mesh-wants.step-ended",
|
||||
"mesh-wants.step-progress",
|
||||
"node-engine.running",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-catalog.version-released",
|
||||
@@ -151,6 +153,18 @@
|
||||
"description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "24h"
|
||||
},
|
||||
"ask-unanswered-bound": {
|
||||
"type": "string",
|
||||
"description": "how long an ask the runner puts on a seat, but a test run's, may go without its holder's first word (an ask-taken, a step progress or the step's end) before it is asked once more, and as long again before its step ends could not run, a Go duration such as 10m (design 47 H H-A-235, H-A-238).",
|
||||
"trusted": false,
|
||||
"default": "10m"
|
||||
},
|
||||
"ask-silent-bound": {
|
||||
"type": "string",
|
||||
"description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).",
|
||||
"trusted": false,
|
||||
"default": "45m"
|
||||
}
|
||||
},
|
||||
"state": [
|
||||
@@ -278,6 +292,22 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:run-untaken-bound}"
|
||||
},
|
||||
{
|
||||
"id": "ask-unanswered-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/ask-unanswered-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:ask-unanswered-bound}"
|
||||
},
|
||||
{
|
||||
"id": "ask-silent-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/ask-silent-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:ask-silent-bound}"
|
||||
},
|
||||
{
|
||||
"id": "widens-read-bound",
|
||||
"type": "file",
|
||||
@@ -326,6 +356,8 @@
|
||||
"MESH_RUNNER_WIDENS_READ_BOUND_FILE": "${dir:state}/widens-read-bound",
|
||||
"MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept",
|
||||
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound",
|
||||
"MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE": "${dir:state}/ask-unanswered-bound",
|
||||
"MESH_RUNNER_ASK_SILENT_BOUND_FILE": "${dir:state}/ask-silent-bound",
|
||||
"MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once",
|
||||
"MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap"
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user