Runner: a pipeline's end waits on no other part (A-472)

The runner keeps each catalog revision it read, bounds every read widens makes to 5 s and says each read's duration, failure or cut in the journal and in could-not-run; runner_cancel reads the pipeline after an ask that did not answer.

Builds: design 47 G A-472
This commit is contained in:
2026-11-17 14:55:27 +01:00
parent 4b75791d3d
commit 871f75acd3
5 changed files with 223 additions and 7 deletions
@@ -47,6 +47,8 @@ type CancelResult struct {
Pipeline string `json:"pipeline"`
Result string `json:"result"`
Refusal string `json:"refusal,omitempty"`
// Note says what an ask that did not answer left: the pipeline read again was cancelled (A-472).
Note string `json:"note,omitempty"`
}
func (in CancelInput) run(list func(seat.ListInput) (seat.ListAnswer, error), ask func(string, any) (json.RawMessage, error)) (any, error) {
@@ -98,7 +100,7 @@ func (in CancelInput) run(list func(seat.ListInput) (seat.ListAnswer, error), as
if in.DryRun {
results = append(results, CancelResult{Pipeline: name, Result: "would-cancel"})
} else if _, err := ask("seat:"+seat.Seat+"."+seat.VerbCancel, seat.CancelInput{Pipeline: name, Why: in.Why}); err != nil {
results = append(results, CancelResult{Pipeline: name, Result: "refused", Refusal: err.Error()})
results = append(results, in.afterUnanswered(name, err, list))
} else {
results = append(results, CancelResult{Pipeline: name, Result: "cancelled"})
}
@@ -108,3 +110,31 @@ func (in CancelInput) run(list func(seat.ListInput) (seat.ListAnswer, error), as
}
return map[string]any{"results": results, "counts": counts}, nil
}
// afterUnanswered reads a pipeline's state after an ask that did not answer (A-472): the verb may have written the end and
// said it before the answer was cut, so a pipeline found cancelled is reported cancelled, with what the ask said; one still
// running, or not read, is refused with both.
func (in CancelInput) afterUnanswered(name string, asked error, list func(seat.ListInput) (seat.ListAnswer, error)) CancelResult {
branch := in.Branch
if branch == "" {
branch = "main"
}
listed := seat.ListInput{}
if in.BelowSource != nil {
listed.Branch = branch
}
all, err := list(listed)
if err != nil {
return CancelResult{Pipeline: name, Result: "refused", Refusal: asked.Error() + "; the pipeline is not read again: " + err.Error()}
}
for _, p := range all.Pipelines {
if p.Pipeline != name {
continue
}
if p.Outcome == seat.PipelineOutcomeCancelled {
return CancelResult{Pipeline: name, Result: "cancelled", Note: "the ask did not answer (" + asked.Error() + "), and the pipeline read again is cancelled"}
}
return CancelResult{Pipeline: name, Result: "refused", Refusal: asked.Error() + "; the pipeline read again is " + p.Outcome}
}
return CancelResult{Pipeline: name, Result: "refused", Refusal: asked.Error() + "; the pipeline is not found when read again"}
}
@@ -67,3 +67,37 @@ func TestCancelTool(t *testing.T) {
})
}
}
// An ask that did not answer is followed by a read of the pipeline: one the verb cancelled before the answer was cut is
// reported cancelled, never refused (A-472); one still running, or not read, stays refused.
func TestCancelReadsThePipelineAfterAnAskThatDidNotAnswer(t *testing.T) {
for _, c := range []struct {
name, outcome, listErr, result string
}{
{"cancelled", seat.PipelineOutcomeCancelled, "", "cancelled"},
{"still running", seat.PipelineOutcomeRunning, "", "refused"},
{"not read", "", "bus down", "refused"},
} {
t.Run(c.name, func(t *testing.T) {
asked := false
list := func(seat.ListInput) (seat.ListAnswer, error) {
if !asked {
return seat.ListAnswer{Pipelines: []seat.Pipeline{{Pipeline: "main.1.a", Branch: "main", Outcome: seat.PipelineOutcomeRunning}}}, nil
}
if c.listErr != "" {
return seat.ListAnswer{}, errors.New(c.listErr)
}
return seat.ListAnswer{Pipelines: []seat.Pipeline{{Pipeline: "main.1.a", Branch: "main", Outcome: c.outcome}}}, nil
}
ask := func(string, any) (json.RawMessage, error) {
asked = true
return nil, errors.New("no answer in 30s")
}
got, err := CancelTool(list, ask).Run(map[string]any{"pipelines": []any{"main.1.a"}, "why": "w"})
raw, _ := json.Marshal(got)
if err != nil || !strings.Contains(string(raw), `"result":"`+c.result+`"`) || !strings.Contains(string(raw), "no answer in 30s") {
t.Fatalf("%s, %v; want %s with the ask's words", raw, err, c.result)
}
})
}
}
@@ -143,7 +143,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }}
hearer.Fills = fills
// A pipeline's end names what its change widens (§71, A-207), computed by this running runner from the catalog and the grants part.
hearer.Widens = widenerOf(calls.Call).Widens
hearer.Widens = widenerOf(calls.Call, logger.Printf).Widens
hearer.LogLast = stepLog{ctx: ctx, js: js, module: cred.Module}.Last
hearer.Where = routerRecordOf(cred.Module, calls.Call, calls.Ask)
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
@@ -4,6 +4,8 @@ import (
"encoding/json"
"fmt"
"strconv"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -17,9 +19,14 @@ import (
// widenerOf is the runner's computation of `widens` (design 47 G §71, A-207, A-215, A-220) over its calls: the catalog seat's
// `revision` for the candidate and the branch's newest, the holders from the seat holders' one reading, and the grants part's
// `grants-for`; no other verb.
func widenerOf(call func(key string, body any) (json.RawMessage, error)) *pipeline.Widener {
//
// A pipeline's end is said without waiting on another part (A-472): each read is bounded to readBound and says in the journal how
// long it took, and a failed or cut one is said there and in the widening's could-not-run; a catalog revision, which never
// changes, is kept once read.
func widenerOf(call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) *pipeline.Widener {
call = boundedCalls(call, readBound, time.After, logf)
return &pipeline.Widener{
Revision: catalogRevision(call),
Revision: catalogRevision(call, newRevisionCache(revisionsKept)),
Holder: func(s string) (string, error) { return seatlist.HolderOf(call, s) },
GrantsFor: func(branch string, source int64) (meshgrants.GrantsFor, error) {
return grantsFor(call, branch, source)
@@ -27,9 +34,89 @@ func widenerOf(call func(key string, body any) (json.RawMessage, error)) *pipeli
}
}
// catalogRevision reads one revision of a branch whole, page by page; revision 0 asks the newest (§195).
func catalogRevision(call func(string, any) (json.RawMessage, error)) func(string, int64) (pipeline.Half, error) {
const (
// readBound is how long one read of widens waits for its answer (A-472): a few seconds, never the call's 30.
readBound = 5 * time.Second
// revisionsKept is how many catalog revisions the runner keeps once read; a revision of the catalog graph is some 1.4 MB.
revisionsKept = 6
)
// boundedCalls bounds each call to bound, said in logf with its duration, and a failed or cut one in words that name the call;
// a call that outlives its bound ends on its own at the connection's timeout, its answer dropped.
func boundedCalls(call func(string, any) (json.RawMessage, error), bound time.Duration, after func(time.Duration) <-chan time.Time,
logf func(string, ...any)) func(string, any) (json.RawMessage, error) {
if logf == nil {
logf = func(string, ...any) {}
}
return func(key string, body any) (json.RawMessage, error) {
type answer struct {
raw json.RawMessage
err error
}
start := time.Now()
done := make(chan answer, 1)
go func() {
raw, err := call(key, body)
done <- answer{raw, err}
}()
select {
case a := <-done:
if a.err != nil {
logf("widens: %s failed after %s: %v", key, time.Since(start).Round(time.Millisecond), a.err)
} else {
logf("widens: %s answered %d bytes in %s", key, len(a.raw), time.Since(start).Round(time.Millisecond))
}
return a.raw, a.err
case <-after(bound):
logf("widens: %s cut after %s, not answered", key, bound)
return nil, fmt.Errorf("%s was cut after %s without an answer", key, bound)
}
}
}
// revisionCache keeps the last catalog revisions read, whole: a revision never changes, so one read is read for good.
type revisionCache struct {
mu sync.Mutex
max int
order []string
kept map[string]pipeline.Half
}
func newRevisionCache(max int) *revisionCache {
return &revisionCache{max: max, kept: map[string]pipeline.Half{}}
}
func (c *revisionCache) get(branch string, revision int64) (pipeline.Half, bool) {
c.mu.Lock()
defer c.mu.Unlock()
h, ok := c.kept[branch+"@"+strconv.FormatInt(revision, 10)]
return h, ok
}
func (c *revisionCache) put(branch string, h pipeline.Half) {
c.mu.Lock()
defer c.mu.Unlock()
k := branch + "@" + strconv.FormatInt(h.Revision, 10)
if _, ok := c.kept[k]; !ok {
c.order = append(c.order, k)
}
c.kept[k] = h
for len(c.order) > c.max {
delete(c.kept, c.order[0])
c.order = c.order[1:]
}
}
// catalogRevision reads one revision of a branch whole, page by page; revision 0 asks the newest (§195). A revision read is
// kept (A-472): one asked by its number is answered from what is kept, and the newest, once its first page names a revision
// that is kept, is answered without the pages after it.
func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revisionCache) func(string, int64) (pipeline.Half, error) {
return func(branch string, revision int64) (pipeline.Half, error) {
if revision > 0 {
if h, ok := kept.get(branch, revision); ok {
return h, nil
}
}
in := catalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision}
var half pipeline.Half
for {
@@ -49,6 +136,9 @@ func catalogRevision(call func(string, any) (json.RawMessage, error)) func(strin
return pipeline.Half{}, fmt.Errorf("%s holds no revision %d of %s", catalog.VerbRevision, revision, branch)
}
if half.Entries == nil {
if h, ok := kept.get(branch, a.Revision.Revision); ok {
return h, nil
}
half = pipeline.Half{Revision: a.Revision.Revision, Source: a.Revision.Source, Entries: map[string]catalog.CatalogEntry{}}
in.Revision = a.Revision.Revision // the pages of one revision, though a newer one is said between them
}
@@ -56,6 +146,7 @@ func catalogRevision(call func(string, any) (json.RawMessage, error)) func(strin
half.Entries[e.Module] = e
}
if a.Next == "" {
kept.put(branch, half)
return half, nil
}
in.After = a.Next
@@ -2,8 +2,12 @@ package main
import (
"encoding/json"
"fmt"
"strings"
"testing"
"time"
"mesh-delivery-pipeline/internal/pipeline"
)
// The runner reads a revision whole, page by page, of the revision the first page named, and the candidate by its number; the grants
@@ -21,7 +25,7 @@ func TestWidenerReadsARevisionWholeAndAsksGrantsFor(t *testing.T) {
}
return json.RawMessage(`{"revision":{"revision":8,"kind":"trunk","entries":[]},"next":"a","entries":[{"module":"a","commit":"c","repository":"r","manifest":{}}]}`), nil
}
half, err := catalogRevision(call)("feat/x", 0)
half, err := catalogRevision(call, newRevisionCache(2))("feat/x", 0)
if err != nil || half.Revision != 8 || len(half.Entries) != 2 {
t.Fatalf("half %+v, %v", half, err)
}
@@ -33,3 +37,60 @@ func TestWidenerReadsARevisionWholeAndAsksGrantsFor(t *testing.T) {
t.Errorf("grants %+v, %v, asked %v", g, err, asked)
}
}
// A revision of the catalog never changes: once read it is kept, the candidate asked by its number is answered without a call,
// and the newest, whose first page names a revision that is kept, is answered without the pages after it; the oldest is dropped
// beyond the bound (A-472).
func TestCatalogRevisionIsKeptOnceRead(t *testing.T) {
calls := 0
call := func(key string, body any) (json.RawMessage, error) {
calls++
raw, _ := json.Marshal(body)
if strings.Contains(string(raw), `"after":"a"`) {
return json.RawMessage(`{"revision":{"revision":8,"kind":"trunk","entries":[]},"entries":[{"module":"b","commit":"c","repository":"r","manifest":{}}]}`), nil
}
return json.RawMessage(`{"revision":{"revision":8,"kind":"trunk","entries":[]},"next":"a","entries":[{"module":"a","commit":"c","repository":"r","manifest":{}}]}`), nil
}
read := catalogRevision(call, newRevisionCache(1))
if _, err := read("main", 0); err != nil || calls != 2 {
t.Fatalf("the first read made %d calls, %v; want its two pages", calls, err)
}
if h, err := read("main", 8); err != nil || calls != 2 || len(h.Entries) != 2 {
t.Fatalf("revision 8 asked by its number made calls %d, %+v, %v; want it from what is kept", calls, h, err)
}
if h, err := read("main", 0); err != nil || calls != 3 || len(h.Entries) != 2 {
t.Fatalf("the newest made calls %d, %+v, %v; want its first page only", calls, h, err)
}
c := newRevisionCache(1)
c.put("main", pipeline.Half{Revision: 1})
c.put("main", pipeline.Half{Revision: 2})
if _, ok := c.get("main", 1); ok {
t.Error("the oldest revision is kept beyond the bound")
}
}
// A read that does not answer within its bound is cut and said, in the journal and in the error the widening names; one that
// answers is said with its duration (A-472).
func TestBoundedCallsCutAReadThatDoesNotAnswer(t *testing.T) {
block := make(chan struct{})
defer close(block)
var said []string
logf := func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
fired := make(chan time.Time, 1)
fired <- time.Time{}
call := func(key string, _ any) (json.RawMessage, error) {
if key == "fast" {
return json.RawMessage(`{}`), nil
}
<-block
return nil, nil
}
if _, err := boundedCalls(call, time.Second, func(time.Duration) <-chan time.Time { return fired }, logf)("slow", nil); err == nil ||
!strings.Contains(err.Error(), "cut after 1s") || len(said) != 1 || !strings.Contains(said[0], "cut after 1s") {
t.Fatalf("slow: %v, said %v", err, said)
}
if _, err := boundedCalls(call, time.Second, func(time.Duration) <-chan time.Time { return nil }, logf)("fast", nil); err != nil ||
len(said) != 2 || !strings.Contains(said[1], "answered 2 bytes in") {
t.Fatalf("fast: %v, said %v", err, said)
}
}