Runner: a pipeline's end waits on no other part (A-472)
The runner keeps each catalog revision it read, bounds every read widens makes to 5 s and says each read's duration, failure or cut in the journal and in could-not-run; runner_cancel reads the pipeline after an ask that did not answer. Builds: design 47 G A-472
This commit is contained in:
@@ -47,6 +47,8 @@ type CancelResult struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Result string `json:"result"`
|
||||
Refusal string `json:"refusal,omitempty"`
|
||||
// Note says what an ask that did not answer left: the pipeline read again was cancelled (A-472).
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
func (in CancelInput) run(list func(seat.ListInput) (seat.ListAnswer, error), ask func(string, any) (json.RawMessage, error)) (any, error) {
|
||||
@@ -98,7 +100,7 @@ func (in CancelInput) run(list func(seat.ListInput) (seat.ListAnswer, error), as
|
||||
if in.DryRun {
|
||||
results = append(results, CancelResult{Pipeline: name, Result: "would-cancel"})
|
||||
} else if _, err := ask("seat:"+seat.Seat+"."+seat.VerbCancel, seat.CancelInput{Pipeline: name, Why: in.Why}); err != nil {
|
||||
results = append(results, CancelResult{Pipeline: name, Result: "refused", Refusal: err.Error()})
|
||||
results = append(results, in.afterUnanswered(name, err, list))
|
||||
} else {
|
||||
results = append(results, CancelResult{Pipeline: name, Result: "cancelled"})
|
||||
}
|
||||
@@ -108,3 +110,31 @@ func (in CancelInput) run(list func(seat.ListInput) (seat.ListAnswer, error), as
|
||||
}
|
||||
return map[string]any{"results": results, "counts": counts}, nil
|
||||
}
|
||||
|
||||
// afterUnanswered reads a pipeline's state after an ask that did not answer (A-472): the verb may have written the end and
|
||||
// said it before the answer was cut, so a pipeline found cancelled is reported cancelled, with what the ask said; one still
|
||||
// running, or not read, is refused with both.
|
||||
func (in CancelInput) afterUnanswered(name string, asked error, list func(seat.ListInput) (seat.ListAnswer, error)) CancelResult {
|
||||
branch := in.Branch
|
||||
if branch == "" {
|
||||
branch = "main"
|
||||
}
|
||||
listed := seat.ListInput{}
|
||||
if in.BelowSource != nil {
|
||||
listed.Branch = branch
|
||||
}
|
||||
all, err := list(listed)
|
||||
if err != nil {
|
||||
return CancelResult{Pipeline: name, Result: "refused", Refusal: asked.Error() + "; the pipeline is not read again: " + err.Error()}
|
||||
}
|
||||
for _, p := range all.Pipelines {
|
||||
if p.Pipeline != name {
|
||||
continue
|
||||
}
|
||||
if p.Outcome == seat.PipelineOutcomeCancelled {
|
||||
return CancelResult{Pipeline: name, Result: "cancelled", Note: "the ask did not answer (" + asked.Error() + "), and the pipeline read again is cancelled"}
|
||||
}
|
||||
return CancelResult{Pipeline: name, Result: "refused", Refusal: asked.Error() + "; the pipeline read again is " + p.Outcome}
|
||||
}
|
||||
return CancelResult{Pipeline: name, Result: "refused", Refusal: asked.Error() + "; the pipeline is not found when read again"}
|
||||
}
|
||||
|
||||
@@ -67,3 +67,37 @@ func TestCancelTool(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An ask that did not answer is followed by a read of the pipeline: one the verb cancelled before the answer was cut is
|
||||
// reported cancelled, never refused (A-472); one still running, or not read, stays refused.
|
||||
func TestCancelReadsThePipelineAfterAnAskThatDidNotAnswer(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, outcome, listErr, result string
|
||||
}{
|
||||
{"cancelled", seat.PipelineOutcomeCancelled, "", "cancelled"},
|
||||
{"still running", seat.PipelineOutcomeRunning, "", "refused"},
|
||||
{"not read", "", "bus down", "refused"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
asked := false
|
||||
list := func(seat.ListInput) (seat.ListAnswer, error) {
|
||||
if !asked {
|
||||
return seat.ListAnswer{Pipelines: []seat.Pipeline{{Pipeline: "main.1.a", Branch: "main", Outcome: seat.PipelineOutcomeRunning}}}, nil
|
||||
}
|
||||
if c.listErr != "" {
|
||||
return seat.ListAnswer{}, errors.New(c.listErr)
|
||||
}
|
||||
return seat.ListAnswer{Pipelines: []seat.Pipeline{{Pipeline: "main.1.a", Branch: "main", Outcome: c.outcome}}}, nil
|
||||
}
|
||||
ask := func(string, any) (json.RawMessage, error) {
|
||||
asked = true
|
||||
return nil, errors.New("no answer in 30s")
|
||||
}
|
||||
got, err := CancelTool(list, ask).Run(map[string]any{"pipelines": []any{"main.1.a"}, "why": "w"})
|
||||
raw, _ := json.Marshal(got)
|
||||
if err != nil || !strings.Contains(string(raw), `"result":"`+c.result+`"`) || !strings.Contains(string(raw), "no answer in 30s") {
|
||||
t.Fatalf("%s, %v; want %s with the ask's words", raw, err, c.result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }}
|
||||
hearer.Fills = fills
|
||||
// A pipeline's end names what its change widens (§71, A-207), computed by this running runner from the catalog and the grants part.
|
||||
hearer.Widens = widenerOf(calls.Call).Widens
|
||||
hearer.Widens = widenerOf(calls.Call, logger.Printf).Widens
|
||||
hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last
|
||||
hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask)
|
||||
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -17,9 +19,14 @@ import (
|
||||
// widenerOf is the runner's computation of `widens` (design 47 G §71, A-207, A-215, A-220) over its calls: the catalog seat's
|
||||
// `revision` for the candidate and the branch's newest, the holders from the seat holders' one reading, and the grants part's
|
||||
// `grants-for`; no other verb.
|
||||
func widenerOf(call func(key string, body any) (json.RawMessage, error)) *pipeline.Widener {
|
||||
//
|
||||
// A pipeline's end is said without waiting on another part (A-472): each read is bounded to readBound and says in the journal how
|
||||
// long it took, and a failed or cut one is said there and in the widening's could-not-run; a catalog revision, which never
|
||||
// changes, is kept once read.
|
||||
func widenerOf(call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) *pipeline.Widener {
|
||||
call = boundedCalls(call, readBound, time.After, logf)
|
||||
return &pipeline.Widener{
|
||||
Revision: catalogRevision(call),
|
||||
Revision: catalogRevision(call, newRevisionCache(revisionsKept)),
|
||||
Holder: func(s string) (string, error) { return seatlist.HolderOf(call, s) },
|
||||
GrantsFor: func(branch string, source int64) (meshgrants.GrantsFor, error) {
|
||||
return grantsFor(call, branch, source)
|
||||
@@ -27,9 +34,89 @@ func widenerOf(call func(key string, body any) (json.RawMessage, error)) *pipeli
|
||||
}
|
||||
}
|
||||
|
||||
// catalogRevision reads one revision of a branch whole, page by page; revision 0 asks the newest (§195).
|
||||
func catalogRevision(call func(string, any) (json.RawMessage, error)) func(string, int64) (pipeline.Half, error) {
|
||||
const (
|
||||
// readBound is how long one read of widens waits for its answer (A-472): a few seconds, never the call's 30.
|
||||
readBound = 5 * time.Second
|
||||
// revisionsKept is how many catalog revisions the runner keeps once read; a revision of the catalog graph is some 1.4 MB.
|
||||
revisionsKept = 6
|
||||
)
|
||||
|
||||
// boundedCalls bounds each call to bound, said in logf with its duration, and a failed or cut one in words that name the call;
|
||||
// a call that outlives its bound ends on its own at the connection's timeout, its answer dropped.
|
||||
func boundedCalls(call func(string, any) (json.RawMessage, error), bound time.Duration, after func(time.Duration) <-chan time.Time,
|
||||
logf func(string, ...any)) func(string, any) (json.RawMessage, error) {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
return func(key string, body any) (json.RawMessage, error) {
|
||||
type answer struct {
|
||||
raw json.RawMessage
|
||||
err error
|
||||
}
|
||||
start := time.Now()
|
||||
done := make(chan answer, 1)
|
||||
go func() {
|
||||
raw, err := call(key, body)
|
||||
done <- answer{raw, err}
|
||||
}()
|
||||
select {
|
||||
case a := <-done:
|
||||
if a.err != nil {
|
||||
logf("widens: %s failed after %s: %v", key, time.Since(start).Round(time.Millisecond), a.err)
|
||||
} else {
|
||||
logf("widens: %s answered %d bytes in %s", key, len(a.raw), time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
return a.raw, a.err
|
||||
case <-after(bound):
|
||||
logf("widens: %s cut after %s, not answered", key, bound)
|
||||
return nil, fmt.Errorf("%s was cut after %s without an answer", key, bound)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// revisionCache keeps the last catalog revisions read, whole: a revision never changes, so one read is read for good.
|
||||
type revisionCache struct {
|
||||
mu sync.Mutex
|
||||
max int
|
||||
order []string
|
||||
kept map[string]pipeline.Half
|
||||
}
|
||||
|
||||
func newRevisionCache(max int) *revisionCache {
|
||||
return &revisionCache{max: max, kept: map[string]pipeline.Half{}}
|
||||
}
|
||||
|
||||
func (c *revisionCache) get(branch string, revision int64) (pipeline.Half, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
h, ok := c.kept[branch+"@"+strconv.FormatInt(revision, 10)]
|
||||
return h, ok
|
||||
}
|
||||
|
||||
func (c *revisionCache) put(branch string, h pipeline.Half) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
k := branch + "@" + strconv.FormatInt(h.Revision, 10)
|
||||
if _, ok := c.kept[k]; !ok {
|
||||
c.order = append(c.order, k)
|
||||
}
|
||||
c.kept[k] = h
|
||||
for len(c.order) > c.max {
|
||||
delete(c.kept, c.order[0])
|
||||
c.order = c.order[1:]
|
||||
}
|
||||
}
|
||||
|
||||
// catalogRevision reads one revision of a branch whole, page by page; revision 0 asks the newest (§195). A revision read is
|
||||
// kept (A-472): one asked by its number is answered from what is kept, and the newest, once its first page names a revision
|
||||
// that is kept, is answered without the pages after it.
|
||||
func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revisionCache) func(string, int64) (pipeline.Half, error) {
|
||||
return func(branch string, revision int64) (pipeline.Half, error) {
|
||||
if revision > 0 {
|
||||
if h, ok := kept.get(branch, revision); ok {
|
||||
return h, nil
|
||||
}
|
||||
}
|
||||
in := catalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision}
|
||||
var half pipeline.Half
|
||||
for {
|
||||
@@ -49,6 +136,9 @@ func catalogRevision(call func(string, any) (json.RawMessage, error)) func(strin
|
||||
return pipeline.Half{}, fmt.Errorf("%s holds no revision %d of %s", catalog.VerbRevision, revision, branch)
|
||||
}
|
||||
if half.Entries == nil {
|
||||
if h, ok := kept.get(branch, a.Revision.Revision); ok {
|
||||
return h, nil
|
||||
}
|
||||
half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}}
|
||||
in.Revision = a.Revision.Revision // the pages of one revision, though a newer one is said between them
|
||||
}
|
||||
@@ -56,6 +146,7 @@ func catalogRevision(call func(string, any) (json.RawMessage, error)) func(strin
|
||||
half.Entries[e.Module] = e
|
||||
}
|
||||
if a.Next == "" {
|
||||
kept.put(branch, half)
|
||||
return half, nil
|
||||
}
|
||||
in.After = a.Next
|
||||
|
||||
@@ -2,8 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// The runner reads a revision whole, page by page, of the revision the first page named, and the candidate by its number; the grants
|
||||
@@ -21,7 +25,7 @@ func TestWidenerReadsARevisionWholeAndAsksGrantsFor(t *testing.T) {
|
||||
}
|
||||
return json.RawMessage(`{"revision":{"revision":8,"kind":"trunk","entries":[]},"next":"a","entries":[{"module":"a","commit":"c","repository":"r","manifest":{}}]}`), nil
|
||||
}
|
||||
half, err := catalogRevision(call)("feat/x", 0)
|
||||
half, err := catalogRevision(call, newRevisionCache(2))("feat/x", 0)
|
||||
if err != nil || half.Revision != 8 || len(half.Entries) != 2 {
|
||||
t.Fatalf("half %+v, %v", half, err)
|
||||
}
|
||||
@@ -33,3 +37,60 @@ func TestWidenerReadsARevisionWholeAndAsksGrantsFor(t *testing.T) {
|
||||
t.Errorf("grants %+v, %v, asked %v", g, err, asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A revision of the catalog never changes: once read it is kept, the candidate asked by its number is answered without a call,
|
||||
// and the newest, whose first page names a revision that is kept, is answered without the pages after it; the oldest is dropped
|
||||
// beyond the bound (A-472).
|
||||
func TestCatalogRevisionIsKeptOnceRead(t *testing.T) {
|
||||
calls := 0
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
calls++
|
||||
raw, _ := json.Marshal(body)
|
||||
if strings.Contains(string(raw), `"after":"a"`) {
|
||||
return json.RawMessage(`{"revision":{"revision":8,"kind":"trunk","entries":[]},"entries":[{"module":"b","commit":"c","repository":"r","manifest":{}}]}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"revision":{"revision":8,"kind":"trunk","entries":[]},"next":"a","entries":[{"module":"a","commit":"c","repository":"r","manifest":{}}]}`), nil
|
||||
}
|
||||
read := catalogRevision(call, newRevisionCache(1))
|
||||
if _, err := read("main", 0); err != nil || calls != 2 {
|
||||
t.Fatalf("the first read made %d calls, %v; want its two pages", calls, err)
|
||||
}
|
||||
if h, err := read("main", 8); err != nil || calls != 2 || len(h.Entries) != 2 {
|
||||
t.Fatalf("revision 8 asked by its number made calls %d, %+v, %v; want it from what is kept", calls, h, err)
|
||||
}
|
||||
if h, err := read("main", 0); err != nil || calls != 3 || len(h.Entries) != 2 {
|
||||
t.Fatalf("the newest made calls %d, %+v, %v; want its first page only", calls, h, err)
|
||||
}
|
||||
c := newRevisionCache(1)
|
||||
c.put("main", pipeline.Half{Revision: 1})
|
||||
c.put("main", pipeline.Half{Revision: 2})
|
||||
if _, ok := c.get("main", 1); ok {
|
||||
t.Error("the oldest revision is kept beyond the bound")
|
||||
}
|
||||
}
|
||||
|
||||
// A read that does not answer within its bound is cut and said, in the journal and in the error the widening names; one that
|
||||
// answers is said with its duration (A-472).
|
||||
func TestBoundedCallsCutAReadThatDoesNotAnswer(t *testing.T) {
|
||||
block := make(chan struct{})
|
||||
defer close(block)
|
||||
var said []string
|
||||
logf := func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
fired := make(chan time.Time, 1)
|
||||
fired <- time.Time{}
|
||||
call := func(key string, _ any) (json.RawMessage, error) {
|
||||
if key == "fast" {
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
<-block
|
||||
return nil, nil
|
||||
}
|
||||
if _, err := boundedCalls(call, time.Second, func(time.Duration) <-chan time.Time { return fired }, logf)("slow", nil); err == nil ||
|
||||
!strings.Contains(err.Error(), "cut after 1s") || len(said) != 1 || !strings.Contains(said[0], "cut after 1s") {
|
||||
t.Fatalf("slow: %v, said %v", err, said)
|
||||
}
|
||||
if _, err := boundedCalls(call, time.Second, func(time.Duration) <-chan time.Time { return nil }, logf)("fast", nil); err != nil ||
|
||||
len(said) != 2 || !strings.Contains(said[1], "answered 2 bytes in") {
|
||||
t.Fatalf("fast: %v, said %v", err, said)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user