H-A-496 runner: say kept revisions; a gone kept revision ends the pipeline
main.291 held the one pipelines-at-once turn 50 minutes waiting for kept revision 292, which the catalog had removed: nothing pinned it and the wait had no end. pipeline-started now carries every revision the pipeline's movers are kept at, and a re-pin says it again with the full list, so the indexer pins them. A catalog read answered "no such object" is the revision gone; the placing then ends the pipeline failed "kept revision <r> gone", its movers are left unfinished and rejoin the next pipeline, and its turn is freed. Builds: design 47 H2 H-A-496, H-A-496 addendum
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -150,10 +151,17 @@ func catalogRevision(call func(string, any) (json.RawMessage, error), kept *revi
|
||||
for {
|
||||
raw, err := call("seat:"+catalog.Seat+"."+catalog.VerbRevision, in)
|
||||
if err != nil {
|
||||
// A revision the catalog answers "no such object" for is gone (design 47 H2 H-A-496): a kept mover's wait on it ends.
|
||||
if strings.Contains(err.Error(), "no such object") {
|
||||
return pipeline.Half{}, fmt.Errorf("%w: %v", pipeline.ErrRevisionGone, err)
|
||||
}
|
||||
return pipeline.Half{}, err
|
||||
}
|
||||
answer, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
if strings.Contains(said, "no such object") {
|
||||
return pipeline.Half{}, fmt.Errorf("%w: %s answered: %s", pipeline.ErrRevisionGone, catalog.VerbRevision, said)
|
||||
}
|
||||
return pipeline.Half{}, fmt.Errorf("%s answered: %s", catalog.VerbRevision, said)
|
||||
}
|
||||
var a catalog.RevisionAnswer
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -63,6 +64,19 @@ func TestCatalogRevisionIsKeptOnceRead(t *testing.T) {
|
||||
if h, err := read("main", 0); err != nil || calls != 3 || len(h.Entries) != 2 {
|
||||
t.Fatalf("the newest made calls %d, %+v, %v; want its first page only", calls, h, err)
|
||||
}
|
||||
// A revision the catalog answers "no such object" for is gone, whether refused or answered so (design 47 H2 H-A-496).
|
||||
for _, gone := range []func(string, any) (json.RawMessage, error){
|
||||
func(string, any) (json.RawMessage, error) {
|
||||
return nil, fmt.Errorf("revision main.292: no such object")
|
||||
},
|
||||
func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"content":[{"text":"revision main.292: no such object"}],"isError":true}`), nil
|
||||
},
|
||||
} {
|
||||
if _, err := catalogRevision(gone, newRevisionCache(1))("main", 292); !errors.Is(err, pipeline.ErrRevisionGone) {
|
||||
t.Errorf("a revision the catalog has no object of reads %v, want it gone", err)
|
||||
}
|
||||
}
|
||||
c := newRevisionCache(1)
|
||||
c.put("main", pipeline.Half{Revision: 1})
|
||||
c.put("main", pipeline.Half{Revision: 2})
|
||||
|
||||
@@ -132,6 +132,11 @@ func (s *Starter) keepIn(key string, half Half, movers []seat.Mover) []seat.Move
|
||||
s.logf("%s keeps the movers revision %d of its own source %d re-pinned, %s: %d steps ended superseded, %d asked again (H-A-283, H-A-284)",
|
||||
key, half.Revision, half.Source, strings.Join(moved, ", "), len(ended), len(asks))
|
||||
s.sayRePinned(p, kept, revisions, half.Revision)
|
||||
// The re-pin says pipeline-started again with every revision its movers are kept at, so the indexer pins them (design
|
||||
// 47 H2 H-A-496 and its addendum).
|
||||
if keeps {
|
||||
s.sayStartedPipeline(p, now)
|
||||
}
|
||||
for _, st := range ended {
|
||||
if s.Say == nil {
|
||||
break
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
asks := &stepAsks{}
|
||||
var said []seat.StepEvent
|
||||
var repinned []string
|
||||
var repinned, starts []string
|
||||
var left [][]seat.StepResult
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: asks, Name: "the-runner", Now: ticking(), Revision: read,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, m == "app" },
|
||||
@@ -77,6 +77,9 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if ev, ok := body.(seat.RePinnedEvent); ok {
|
||||
repinned = append(repinned, fmt.Sprintf("%s %s %s %d", subject, ev.Pipeline, ev.Mover, ev.Revision))
|
||||
}
|
||||
if ev, ok := body.(seat.PipelineStarted); ok {
|
||||
starts = append(starts, fmt.Sprintf("%s %v", ev.Pipeline, ev.Kept))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Left: func(_ string, leave, drop []seat.StepResult, _ string) { left = append(left, append(leave, drop...)) }}
|
||||
@@ -133,6 +136,11 @@ func TestASameSourceRepinKeepsTheMoverInItsPipeline(t *testing.T) {
|
||||
if kept, err := store.KeptRevisions(key); err != nil || kept["app"] != 11 {
|
||||
t.Errorf("kept-revisions = %v (%v), want app at 11", kept, err)
|
||||
}
|
||||
// design 47 H2 H-A-496 and its addendum: the start names no kept revision, the re-pin says pipeline-started again with
|
||||
// every revision its movers are kept at.
|
||||
if want := []string{key + " []", key + " [11]"}; !slices.Equal(starts, want) {
|
||||
t.Errorf("pipeline-started said %q, want %q", starts, want)
|
||||
}
|
||||
// The pin check reads the kept mover's entry at the revision it was kept at, never the pipeline's.
|
||||
if n, err := s.AskPinHeld(store, key, "app", map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.26", "git.novox.be/novox/x/go": "v0.1.1"}, read); n == 0 || err != nil {
|
||||
after, _, _ := store.Get(key)
|
||||
@@ -255,6 +263,60 @@ func TestAKeptMoverWaitingOnItsRevisionSaysSo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 H2 H-A-496: a kept mover whose kept revision the catalog answers "no such object" for waits no longer: the
|
||||
// pipeline ends failed "kept revision 11 gone" at the placing, is said ended, frees its turn, and leaves its mover
|
||||
// unfinished, so it rejoins the next pipeline.
|
||||
func TestAKeptRevisionGoneEndsThePipeline(t *testing.T) {
|
||||
reg, entries, _, read := repinFixture()
|
||||
revision := func(branch string, rev int64) (Half, error) {
|
||||
if rev == 11 {
|
||||
return Half{}, fmt.Errorf("revision main.11: no such object: %w", ErrRevisionGone)
|
||||
}
|
||||
return read(branch, rev)
|
||||
}
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
var turned, ended []string
|
||||
u := memUnfinished{}
|
||||
h := &Hearer{Store: store, Name: "the-runner", Unfinished: &Unfinished{Store: u},
|
||||
Say: func(subject string, body any) error {
|
||||
if ev, ok := body.(seat.PipelineEnded); ok {
|
||||
ended = append(ended, ev.Pipeline+" "+ev.Outcome)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Turn: func(key string) { turned = append(turned, key) }}
|
||||
registered := false
|
||||
s := &Starter{Pipelines: store, Store: store, Steps: &stepAsks{}, Name: "the-runner", Now: ticking(), Revision: revision,
|
||||
Say: func(string, any) error { return nil }, Ended: h.EndedAtPlacing,
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
|
||||
key := Key("main", 10)
|
||||
if ok, err := s.Start(Begin{Branch: "main", Source: 41, Revision: 10, Entries: entries,
|
||||
Movers: []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa", Repository: "o/app"}}}); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
if n, err := s.Revised(revisionEvent(11), read); n != 0 || err != nil {
|
||||
t.Fatalf("the re-pin started %d (%v)", n, err)
|
||||
}
|
||||
registered = true
|
||||
lagging := func(string) (Half, bool) { return Half{Revision: 10, Source: 41, Entries: entries}, true }
|
||||
if _, err := s.Registered(store, reg, lagging); err != nil {
|
||||
t.Fatalf("the placing: %v", err)
|
||||
}
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Outcome != seat.PipelineOutcomeFailed || p.Ended == nil || !strings.HasPrefix(p.Why, "kept revision 11 gone") {
|
||||
t.Fatalf("the pipeline = %s, %q, ended %v: want failed with kept revision 11 gone", p.Outcome, p.Why, p.Ended)
|
||||
}
|
||||
if !slices.Equal(ended, []string{key + " failed"}) || !slices.Equal(turned, []string{key}) {
|
||||
t.Fatalf("said ended %v, turns freed %v: want %s's end said and its turn freed", ended, turned, key)
|
||||
}
|
||||
if _, held := u[UnfinishedKey("main", "app")]; !held {
|
||||
t.Fatalf("app is not left unfinished, so it rejoins no pipeline: %v", u)
|
||||
}
|
||||
}
|
||||
|
||||
// rawKept reads a memRaw as the reads do.
|
||||
type rawKept struct{ r *memRaw }
|
||||
|
||||
|
||||
@@ -347,6 +347,27 @@ func (s *Starter) holderless(p *seat.Pipeline, asks []Asked, now time.Time) ([]A
|
||||
return nil, ended
|
||||
}
|
||||
|
||||
// keptGone ends p failed at its placing, "kept revision <r> gone", when the catalog no longer holds revision r a mover of
|
||||
// it is kept at (design 47 H2 H-A-496): each step not ended ends not asked, so its movers, whose steps never ran, are left
|
||||
// unfinished and rejoin the next pipeline (§786), and its end frees its turn under pipelines-at-once. It answers the steps
|
||||
// it ended, none when it had none; the pipeline's Ended says it ended.
|
||||
func keptGone(p *seat.Pipeline, r int64, err error, now time.Time) []int {
|
||||
why := fmt.Sprintf("kept revision %d gone", r)
|
||||
ended := []int{}
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if !open(*st) && st.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
st.Outcome, st.Why, st.Ended, st.Ask, st.Shared = seat.StepResultOutcomeNotAsked, "the pipeline failed: "+why, &now, "", nil
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeFailed, fmt.Sprintf("%s (%v)", why, err), &now
|
||||
p.EndCount++
|
||||
return ended
|
||||
}
|
||||
|
||||
// endedAtPlacing says a pipeline holderless ended at placing (A-470h): each step it ended in its step-ended, then the
|
||||
// pipeline's end, through Ended; with none given it is said in the journal alone.
|
||||
func (s *Starter) endedAtPlacing(p seat.Pipeline, steps []int) error {
|
||||
@@ -416,18 +437,36 @@ func (s *Starter) create(key string, p seat.Pipeline) error {
|
||||
return s.Pipelines.Create(key, raw)
|
||||
}
|
||||
|
||||
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers.
|
||||
func started(p seat.Pipeline, at time.Time) seat.PipelineStarted {
|
||||
// started is a pipeline's pipeline-started (§746): its key, branch, source, call, revision and movers, and kept, every
|
||||
// catalog revision its movers are kept at (design 47 H2 H-A-496), so the indexer pins each while the pipeline runs.
|
||||
func started(p seat.Pipeline, at time.Time, kept []int64) seat.PipelineStarted {
|
||||
return seat.PipelineStarted{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call,
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run}
|
||||
Revision: p.Revision, Movers: p.Movers, Published: p.Published, Run: p.Run, Kept: kept}
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746).
|
||||
// keptList is every distinct revision of kept-revisions, in order (H-A-496).
|
||||
func keptList(kept map[string]int64) []int64 {
|
||||
var out []int64
|
||||
for _, r := range kept {
|
||||
if r > 0 && !slices.Contains(out, r) {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// sayStartedPipeline says p's pipeline-started on <branch>.<revision> (§746), with the revisions its movers are kept at as
|
||||
// its record's kept-revisions holds them (H-A-496): a re-pin says it again with the full list (H-A-496 addendum).
|
||||
func (s *Starter) sayStartedPipeline(p seat.Pipeline, at time.Time) {
|
||||
if s.Say == nil {
|
||||
return
|
||||
}
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at)); err != nil {
|
||||
kept, err := keptOfStore(s.Store, p.Pipeline)
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its pipeline-started names no kept revision: %v", p.Pipeline, err)
|
||||
}
|
||||
if err := s.Say(runnerclient.RevisionSubject(seat.EventPipelineStarted, p.Branch, subjectRevision(p)), started(p, at, keptList(kept))); err != nil {
|
||||
s.logf("pipeline-started of %s is not said: %v", p.Pipeline, err)
|
||||
}
|
||||
}
|
||||
@@ -445,6 +484,8 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
if err != nil {
|
||||
s.logf("the kept-revisions of %s are not read, so its movers are placed as movers not kept: %v", p.Pipeline, err)
|
||||
}
|
||||
var gone int64
|
||||
var goneWhy error
|
||||
all := entries
|
||||
for i, m := range p.Movers {
|
||||
// A mover that left for a newer pipeline is placed there, never here (H-A-223, H-A-361).
|
||||
@@ -460,7 +501,13 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
entries := all
|
||||
if rev := kept[m.Module]; rev > 0 {
|
||||
var err error
|
||||
if entries, err = s.keptEntries(p.Branch, rev); err != nil {
|
||||
if entries, err = s.keptEntries(p.Branch, rev); errors.Is(err, ErrRevisionGone) {
|
||||
// A kept revision the catalog no longer holds is never read: the pipeline ends failed (H-A-496).
|
||||
if gone == 0 {
|
||||
gone, goneWhy = rev, err
|
||||
}
|
||||
continue
|
||||
} else if err != nil {
|
||||
// It waits for its kept revision to be read, said once and carried on the mover (design 47 H2 §787).
|
||||
if m.Unplaced != seat.MoverUnplacedKeptRevisionUnread || m.Revision != rev {
|
||||
s.logf("%s: %s waits for its kept revision %d to be read (%v)", p.Pipeline, m.Module, rev, err)
|
||||
@@ -520,6 +567,9 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
}
|
||||
}
|
||||
}
|
||||
if gone > 0 {
|
||||
return nil, unregistered, keptGone(p, gone, goneWhy, now), notAsked
|
||||
}
|
||||
if steps, ended := cycled(p, waits, unregistered, reg, all, now, s.Name); ended {
|
||||
return nil, unregistered, steps, notAsked
|
||||
} else {
|
||||
@@ -702,7 +752,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
}
|
||||
s.ran(key)
|
||||
s.sayNotAsked(p, notAsked)
|
||||
if len(ended) > 0 {
|
||||
if len(ended) > 0 || p.Ended != nil {
|
||||
return true, s.endedAtPlacing(p, ended)
|
||||
}
|
||||
if s.Steps != nil && len(asks) > 0 {
|
||||
@@ -878,16 +928,19 @@ func (s *Starter) placeWaiting(store Updater, due func(seat.Pipeline) bool, reg
|
||||
if err != nil {
|
||||
note(err)
|
||||
}
|
||||
if n == 0 {
|
||||
// A pipeline its placing ended (a kept revision gone, H-A-496) is said ended though it placed no step.
|
||||
if n == 0 && ended == nil {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
s.logf("%s: %d steps placed from %s (A-327)", key, n, from)
|
||||
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
|
||||
// said already, is said again harmlessly.
|
||||
s.sayStartedPipeline(p, now)
|
||||
if n > 0 {
|
||||
s.logf("%s: %d steps placed from %s (A-327)", key, n, from)
|
||||
// A pipeline whose steps were never asked is started by this placing (A-446a); the pipeline-started of its start,
|
||||
// said already, is said again harmlessly.
|
||||
s.sayStartedPipeline(p, now)
|
||||
}
|
||||
s.sayNotAsked(p, notAsked)
|
||||
if len(ended) > 0 {
|
||||
if ended != nil {
|
||||
if err := s.endedAtPlacing(p, ended); err != nil {
|
||||
note(err)
|
||||
}
|
||||
@@ -933,7 +986,7 @@ func (s *Starter) placeOne(store Updater, key string, due func(seat.Pipeline) bo
|
||||
s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m)
|
||||
s.registrationMissing(m, key)
|
||||
}
|
||||
if len(p.Steps) == before {
|
||||
if len(p.Steps) == before && p.Ended == nil {
|
||||
// A mover that now waits for its kept revision, or no longer does, is written so (design 47 H2 §787).
|
||||
if unplacedMarks(p) == marks {
|
||||
return p, nil, nil, nil, 0, nil
|
||||
|
||||
Reference in New Issue
Block a user