§88: a restarted runner asks the steps placed and not yet asked

Starter.Resume asks again each running step of a branch's running pipelines under the id it was placed under; the branch
view tells Views.OnReady once when it holds its first half, and the runner resumes that branch then.

Builds: design 47 G §22, §24, §88

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
2026-11-11 23:40:22 +01:00
parent f067f2cdb0
commit db942e9204
6 changed files with 170 additions and 4 deletions
@@ -11,6 +11,7 @@ import (
"log"
"os"
"os/signal"
"sync/atomic"
"syscall"
"github.com/nats-io/nats.go"
@@ -60,7 +61,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
hub := process.NewEventHub(logger.Printf)
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, logger.Printf)
// A branch's view holding its first half asks again the steps placed in its running pipelines that were never asked
// (§88); the starter is made after the views, so it is reached through a pointer set once it is.
var resumer atomic.Pointer[func(string)]
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, func(branch string) {
if f := resumer.Load(); f != nil {
(*f)(branch)
}
}, logger.Printf)
if err != nil {
return err
}
@@ -80,6 +88,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// for its step.
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module}
resume := func(branch string) {
half, ok := rv.Catalog.View(branch)
if !ok {
return
}
n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half)
if err != nil {
logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err)
return
}
logger.Printf("the steps placed on %s and not yet asked: %d asked again", branch, n)
}
resumer.Store(&resume)
if err := hearStepEnded(hub, hearer, logger.Printf); err != nil {
return err
}
@@ -25,7 +25,7 @@ type runnerViews struct {
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
// after, then start hub once.
func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State,
call meshcatalogclient.Call, logf func(string, ...any)) (*runnerViews, error) {
call meshcatalogclient.Call, onReady func(branch string), logf func(string, ...any)) (*runnerViews, error) {
findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf)
if err != nil {
return nil, err
@@ -37,7 +37,7 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
// A watched branch first named by a registration gets its view of the catalog half (CT10), adopted through the reader (design
// 47 G §532, A-26): the reader keeps its status, raises and clears its findings through the module's writer, hears the
// seat's catalog-revision on the module's consumer and asks its first view at once.
cv := &pipeline.Views{Logf: logf, Open: func(branch string) (pipeline.BranchView, error) {
cv := &pipeline.Views{Logf: logf, OnReady: onReady, Open: func(branch string) (pipeline.BranchView, error) {
c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, nil)
if err != nil {
return nil, err
@@ -59,6 +59,8 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cr
// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it.
type catalogView struct{ c *meshcatalogclient.Catalog }
func (v catalogView) Ready() <-chan struct{} { return v.c.Ready() }
func (v catalogView) Hear(ctx context.Context, data []byte) error { return v.c.Hear(ctx, data) }
func (v catalogView) View() (pipeline.Half, bool) {
@@ -172,3 +172,45 @@ func TestCompositionIsAskedOfTheAssignerAndItsEndIsTaken(t *testing.T) {
t.Fatalf("decoded end = %+v, %v", ev, err)
}
}
// A runner restarted after it wrote a pipeline and before its asks reached the queues asks them again when the branch's
// view holds its half: each step placed and still running, of a running pipeline of that branch, under the id it was
// placed under; an ended step, an ended pipeline and another branch's pipeline are asked of no one (design 47 G §88).
func TestResumeAsksTheStepsPlacedAndNotEnded(t *testing.T) {
reg, entries := placeFixture()
placed := func(branch string, source int64) seat.Pipeline {
p := seat.Pipeline{Pipeline: Key(branch, source, "app"), Branch: branch, Source: source, Module: "app", Commit: "c1", Fingerprint: "src2:aa",
Repository: "o/app", Outcome: seat.PipelineOutcomeRunning}
p.Steps, _ = Place(p, reg, entries, time.Unix(1, 0), "the-runner")
return p
}
main4, main5, beta6 := placed("main", 4), placed("main", 5), placed("beta", 6)
main5.Steps[1].Outcome = seat.StepResultOutcomePassed // the build of main.5 ended
ended := placed("main", 3)
ended.Outcome = seat.PipelineOutcomePassed
store := memStore{}
for _, p := range []seat.Pipeline{main4, main5, beta6, ended} {
store[p.Pipeline] = struct {
p seat.Pipeline
rev uint64
}{p, 1}
}
asks := &stepAsks{}
s := &Starter{Steps: asks, Name: "the-runner", Registration: func(string) (catalog.Registration, bool) { return reg, true }}
n, err := s.Resume(store, "main", Half{Entries: entries})
// main.4 has the build and three running steps of the fixture; main.5 one fewer, its build ended.
if err != nil || n != 7 || len(asks.asks) != 7 {
t.Fatalf("resume asked %d (%d put), %v; want 7", n, len(asks.asks), err)
}
for _, id := range asks.ids {
found := false
for _, p := range []seat.Pipeline{main4, main5} {
for _, st := range p.Steps {
found = found || (st.Ask == id && st.Outcome == seat.StepResultOutcomeRunning)
}
}
if !found {
t.Errorf("ask %s is of no running step of the branch's running pipelines", id)
}
}
}
@@ -81,6 +81,9 @@ type BranchView interface {
View() (Half, bool)
}
// readyView is a BranchView that says when it holds its first half, as meshcatalogclient.Catalog does.
type readyView interface{ Ready() <-chan struct{} }
// Half is the catalog half a branch view holds, as the runner reads it: its revision and its entries by module.
type Half struct {
Revision int64
@@ -95,7 +98,10 @@ type Views struct {
// Open makes and starts the view of a branch; its error is said in the journal and the branch asked again at the
// next registration heard.
Open func(branch string) (BranchView, error)
Logf func(string, ...any)
// OnReady is told once for a branch, when its view holds a first half: the steps placed in the pipelines of that branch
// and not yet asked are asked then (§88's restart pickup).
OnReady func(branch string)
Logf func(string, ...any)
mu sync.Mutex
branches map[string]BranchView
@@ -126,6 +132,13 @@ func (v *Views) ensure(branch string) {
return
}
v.branches[branch] = bv
if r, ok := bv.(readyView); ok && v.OnReady != nil {
on := v.OnReady
go func() {
<-r.Ready()
on(branch)
}()
}
}
// Branches are the watched branches the views hold, sorted.
@@ -83,3 +83,33 @@ type fakeView struct {
func (f fakeView) Hear(context.Context, []byte) error { f.heard[f.branch]++; return nil }
func (f fakeView) View() (Half, bool) { return Half{Revision: 7}, true }
type readyFake struct {
fakeView
ch chan struct{}
}
func (r readyFake) Ready() <-chan struct{} { return r.ch }
// A branch's view that holds its first half tells Views once, so the runner asks the steps placed on that branch (§88).
func TestViewsTellOnceWhenABranchHoldsItsFirstHalf(t *testing.T) {
ch := make(chan struct{})
told := make(chan string, 4)
v := &Views{OnReady: func(b string) { told <- b }, Open: func(b string) (BranchView, error) {
return readyFake{fakeView{branch: b, heard: map[string]int{}}, ch}, nil
}}
v.Registered(catalog.Registration{Module: "n", Branches: []string{"main"}})
v.Registered(catalog.Registration{Module: "m", Branches: []string{"main"}})
select {
case b := <-told:
t.Fatalf("told %s before the view held a half", b)
default:
}
close(ch)
if b := <-told; b != "main" {
t.Errorf("told %q, want main", b)
}
if len(told) != 0 {
t.Error("a branch's readiness is told once")
}
}
@@ -111,6 +111,64 @@ func (s *Starter) Start(b Begin) (bool, error) {
return true, nil
}
// Resume asks again, of the build seats and the assigner, each step of branch's running pipelines that is placed and still
// running (design 47 G §88's restart pickup): a pipeline written whose asks never reached their queues (the runner stopped
// between its write and its asks, or a publish failed) is asked at the next start. Each ask carries the id its step was
// placed under, so one the seat holds already is kept once. half is the branch's catalog half, read for the steps' artifacts
// and seats as Place reads it. It says how many steps it asked.
func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
if s.Steps == nil || s.Registration == nil {
return 0, nil
}
pkeys, err := store.Keys("*.*.*")
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
asked := 0
var first error
for _, key := range pkeys {
p, _, err := store.Get(key)
if err != nil {
if first == nil {
first = fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
continue
}
if p.Branch != branch || p.Outcome != seat.PipelineOutcomeRunning {
continue
}
reg, ok := s.Registration(p.Module)
if !ok {
continue
}
running := map[string]bool{}
for _, st := range p.Steps {
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
running[st.Ask] = true
}
}
_, placed := Place(p, reg, half.Entries, now, s.Name)
var due []Asked
for _, a := range placed {
if running[a.ID] {
due = append(due, a)
}
}
if len(due) == 0 {
continue
}
if err := askAll(s.Steps, key, due); err != nil && first == nil {
first = err
}
asked += len(due)
}
return asked, first
}
func (s *Starter) logf(format string, args ...any) {
if s.Logf != nil {
s.Logf(format, args...)