R3: the runner says no pinned or freed; a pipeline's pins are the indexer's

The runner's pins of a pipeline's catalog revision (pins.go, PinRunning at its
start, the pin at a start, a rerun and an ask again, the free at an end) go;
the indexer pins on the take-in's write and on pipeline-started, and frees on
pipeline-ended (its half of §749, written by the indexer's developer).

Builds: design 47 H §749, §763, §764

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:31 +01:00
parent def28437ce
commit 3ad2ac455a
7 changed files with 9 additions and 177 deletions
@@ -130,13 +130,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf)
hearer.Asker = busAsker{conn: conn, cred: cred}
// A running pipeline keeps the catalog graph revision it runs on (A-449): said again at each start, as the gate states its
// pins, for pins said before the indexer's consumer existed.
if n, err := hearer.PinRunning(); err != nil {
logger.Printf("the running pipelines' pins are not said: %v", err)
} else {
logger.Printf("pinned the catalog graph revision of %d running pipelines", n)
}
hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf)
// The stores' fill is said before a store is full, and a full one refuses a start (§366).
fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf,
@@ -1,67 +0,0 @@
package pipeline
import (
"strconv"
delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// PinRecord is the record id a pipeline pins its catalog graph revision under: `pipeline.<key>` (design 47 G §153, §579,
// A-449), as a delivery record pins its revisions under its own id, so the indexer keeps the revision while any pipeline
// runs on it.
func PinRecord(key string) string { return "pipeline." + key }
// pinned says whether a pipeline pins a revision: a branch's pipeline that names its catalog graph revision. A run on
// request reads its candidate's revision, which the indexer keeps while the candidate is open, never a branch's.
func pinned(p seat.Pipeline) bool { return p.Run == "" && p.Branch != "" && p.Revision > 0 }
// sayPin says the seat mesh-delivery's pinned or freed for p's catalog graph revision, in the form the gate says them and
// the indexer applies them (a set of record ids per revision, so a repeat changes nothing).
func sayPin(say func(subject string, body any) error, subject string, p seat.Pipeline) error {
if say == nil || !pinned(p) {
return nil
}
return say(subject, delivery.Pins{Record: PinRecord(p.Pipeline), Milestone: "pipeline " + strconv.FormatInt(p.Source, 10),
Catalog: []delivery.PinsCatalogItem{{Branch: p.Branch, Revision: p.Revision}}})
}
// Pin says pinned for the catalog graph revision of a running pipeline (A-449).
func (s *Starter) pin(p seat.Pipeline) {
if err := sayPin(s.Say, delivery.SubjectPinned, p); err != nil {
s.logf("%s does not pin its catalog graph revision %d, so the indexer may remove it: %v", p.Pipeline, p.Revision, err)
}
}
// pin and free say it for a pipeline the hearer reopens or ends (A-449).
func (h *Hearer) pin(p seat.Pipeline) {
if err := sayPin(h.Say, delivery.SubjectPinned, p); err != nil {
h.logf("%s does not pin its catalog graph revision %d, so the indexer may remove it: %v", p.Pipeline, p.Revision, err)
}
}
func (h *Hearer) free(p seat.Pipeline) {
if err := sayPin(h.Say, delivery.SubjectFreed, p); err != nil {
h.logf("%s does not free its catalog graph revision %d, which the indexer keeps while it is pinned: %v", p.Pipeline, p.Revision, err)
}
}
// PinRunning says pinned again for every running pipeline at the runner's start, as the gate states its pins at each
// start: pins said before the indexer's consumer existed, or older than the stream keeps, are heard again, and the set
// form absorbs the repeats. It says how many it said.
func (h *Hearer) PinRunning() (int, error) {
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return 0, err
}
n := 0
for _, key := range pkeys {
p, _, err := h.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || !pinned(p) {
continue
}
h.pin(p)
n++
}
return n, nil
}
@@ -1,7 +1,6 @@
package pipeline
import (
"errors"
"testing"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -9,78 +8,33 @@ import (
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
type saidPin struct {
subject string
pins delivery.Pins
}
// A-449 (design 47 G §153, §579): a pipeline pins its catalog graph revision as a delivery record does, under the record id
// pipeline.<key>, when it starts, again at each start of the runner while it runs, and frees it when it ends, by any
// outcome (superseded included); a run on request, which reads its candidate's revision, pins none.
func TestAPipelinePinsItsCatalogRevisionWhileItRuns(t *testing.T) {
var said []saidPin
// R3/AC1 (design 47 H §749, §763): a pipeline's pins are the indexer's. The runner says no pinned and no freed: not at a
// pipeline's start, its end, its supersession or a rerun.
func TestTheRunnerSaysNoPin(t *testing.T) {
var pins []string
say := func(subject string, body any) error {
if p, ok := body.(delivery.Pins); ok {
said = append(said, saidPin{subject, p})
if _, ok := body.(delivery.Pins); ok || subject == delivery.SubjectPinned || subject == delivery.SubjectFreed {
pins = append(pins, subject)
}
return nil
}
want := func(i int, subject, record string, revision int64) {
t.Helper()
if i >= len(said) {
t.Fatalf("said %d pins, want one more (%s %s %d)", len(said), subject, record, revision)
}
p := said[i]
if p.subject != subject || p.pins.Record != record || len(p.pins.Catalog) != 1 || p.pins.Catalog[0].Branch != "main" || p.pins.Catalog[0].Revision != revision {
t.Errorf("pin %d = %+v, want %s of %s at main %d", i, p, subject, record, revision)
}
}
// At its start.
s := &Starter{Pipelines: created{}, Say: say}
if ok, err := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); !ok || err != nil {
t.Fatalf("start: %v %v", ok, err)
}
want(0, delivery.SubjectPinned, "pipeline.main.7.a", 12)
// A start that names no revision pins nothing; one written already says nothing more.
said = nil
if _, err := s.Start(Begin{Branch: "main", Source: 8, Module: "a", Commit: "c1"}); err != nil || len(said) != 0 {
t.Errorf("a start with no revision said %v (%v)", said, err)
}
if ok, _ := s.Start(Begin{Branch: "main", Source: 7, Module: "a", Commit: "c1", Revision: 12}); ok || len(said) != 0 {
t.Errorf("a start written already said %v", said)
}
// At the runner's start, for the running pipelines alone; a run on request pins none.
h, store, _, _, _, _ := fixture()
h.Say = say
p := store[pkey]
p.p.Revision = 12
store[pkey] = p
store["main.4.old"] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: "main.4.old", Branch: "main", Revision: 11, Outcome: seat.PipelineOutcomePassed}, 1}
store["main.r1.app"] = struct {
p seat.Pipeline
rev uint64
}{seat.Pipeline{Pipeline: "main.r1.app", Branch: "main", Revision: 13, Run: "r1", Outcome: seat.PipelineOutcomeRunning}, 1}
said = nil
if n, err := h.PinRunning(); n != 1 || err != nil {
t.Fatalf("PinRunning said %d (%v), want the running pipeline's alone", n, err)
}
want(0, delivery.SubjectPinned, "pipeline."+pkey, 12)
// Its end frees it, and so does a supersede.
said = nil
if err := h.said(store[pkey].p, false, true); err != nil {
t.Fatal(err)
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
said = nil
if n, err := h.Supersede(seat.Pipeline{Pipeline: "main.6.app", Branch: "main", Source: 6, Module: "app", Commit: "c2", Fingerprint: "src2:bb", Outcome: seat.PipelineOutcomeRunning}, nil); n != 1 || err != nil {
t.Fatalf("supersede %d (%v)", n, err)
}
want(0, delivery.SubjectFreed, "pipeline."+pkey, 12)
if len(said) != 1 {
t.Errorf("a supersede said %d pins, want the one freed (through its end, once)", len(said))
if len(pins) != 0 {
t.Errorf("the runner said %v, want no pinned and no freed", pins)
}
}
@@ -180,44 +134,3 @@ func TestPlacingAWaitingPipelineAsksNoRunningStepAgain(t *testing.T) {
t.Errorf("an advance placed %d (%v) and asked %d steps again, want nothing", n, err, len(asks.asks)-before)
}
}
// A-449: the pipeline-ended's widens read the pipeline's revision while it is still pinned; the pin is freed after the end is
// said, also when saying it fails.
func TestThePinIsFreedAfterTheEndAndItsWidensAreRead(t *testing.T) {
h, store, _, _, _, _ := fixture()
p := store[pkey]
p.p.Revision = 12
store[pkey] = p
freed, readWhileFreed, bodies := false, false, 0
h.Say = func(subject string, body any) error {
switch body.(type) {
case delivery.Pins:
freed = freed || subject == delivery.SubjectFreed
default:
bodies++
}
return nil
}
h.Widens = func(seat.Pipeline) []seat.Widening {
readWhileFreed = freed // the revision is gone once the freed is said
return nil
}
if err := h.said(store[pkey].p, false, true); err != nil {
t.Fatal(err)
}
if readWhileFreed || !freed || bodies == 0 {
t.Errorf("widens read after the freed %v, freed %v, %d other events: want the widens read first and the pin freed after the end", readWhileFreed, freed, bodies)
}
// An end that cannot be said still frees the pin.
freed = false
h.Say = func(subject string, body any) error {
if _, ok := body.(delivery.Pins); ok {
freed = freed || subject == delivery.SubjectFreed
return nil
}
return errors.New("bus away")
}
if err := h.said(store[pkey].p, false, true); err == nil || !freed {
t.Errorf("an end not said: err %v, freed %v, want the error and the pin freed", err, freed)
}
}
@@ -248,7 +248,6 @@ func (s *Starter) Start(b Begin) (bool, error) {
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
s.pin(p) // the catalog graph revision it runs on is kept while it runs (A-449)
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module,
Commit: b.Commit, Fingerprint: b.Fp, Group: b.Group, Revision: b.Revision, Published: b.Published}
commitOrGroup := b.Commit
@@ -31,7 +31,7 @@ func TestStarts(t *testing.T) {
var superseding []string
s := &Starter{Pipelines: store, Say: func(subject string, body any) error {
ev, started := body.(seat.PipelineStarted)
if !started { // a pipeline's pin of its catalog graph revision (A-449) is no pipeline-started
if !started { // only pipeline-started is counted here
return nil
}
said = append(said, subject+" "+ev.Pipeline)
@@ -374,9 +374,6 @@ func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
}
}
if ended {
// The end is said first, with its widens, which read the revision the pipeline pinned; only then is the pin freed (A-449),
// and also when saying the end fails.
defer h.free(p)
if err := h.say(seat.EventPipelineEnded, p, nil); err != nil {
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
}
@@ -287,7 +287,6 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
if err != nil {
return seat.RerunAnswer{}, err
}
v.Hearer.pin(p) // a pipeline that runs again keeps its revision again (A-449)
if ask != nil {
v.Starter.carry(in.Pipeline, []Asked{*ask})
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, v.Starter.recovered(in.Pipeline, p.Commit)); err != nil {
@@ -386,7 +385,6 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
if err != nil {
return seat.RerunAnswer{}, err
}
h.pin(p)
switch {
case join || read:
v.Starter.joinRun(run, reader)
@@ -592,6 +590,5 @@ func (v Verbs) AskAgain(caller string, in seat.AskAgainInput) (seat.AskAgainAnsw
if err != nil {
return seat.AskAgainAnswer{}, err
}
v.Hearer.pin(p)
return seat.AskAgainAnswer{Result: p.Steps[i]}, nil
}