Design 47 H1: the pipelines store guards its shared maps, so the runner's start no longer crashes (H-A-187)
The runner crashed at every start on a live bus with "concurrent map writes" in MoverStore.Get: the waiting pipelines of each branch are placed from goroutines of their own, and the store's last reads and inline marks were written under each pipeline's own lock alone. The store's mutex now guards the lock table, the last reads and the inline marks; behaviour is unchanged. The run test starts the runner over six kept pipelines in the old form, one of 437 steps, so the start path runs under the race detector. Builds: design 47 H H-A-187, H §774
This commit is contained in:
@@ -72,6 +72,9 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Kept pipelines in the old form, their steps inline in the record, as the runner's start finds them on a live bus
|
||||
// (main.41-46, one of 437 steps): each start path reads them at once from its own goroutine (H-A-187, H §774).
|
||||
keptPipelines(t, ctx, js, module)
|
||||
// The step log, the runner's old logs section (§470): the stream the bus makes for it.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -340,3 +343,36 @@ func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bo
|
||||
}
|
||||
t.Fatalf("it did not come about in time; the runner said:\n%s", logs.String())
|
||||
}
|
||||
|
||||
// keptPipelines writes six ended pipelines of main, main.41 to main.46, in the record form before H-A-187: every step inline,
|
||||
// main.46 with 437 of them.
|
||||
func keptPipelines(t *testing.T, ctx context.Context, js jetstream.JetStream, module string) {
|
||||
t.Helper()
|
||||
kv, err := js.KeyValue(ctx, module+"_pipelines")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Date(2026, 11, 20, 23, 0, 0, 0, time.UTC)
|
||||
for source := int64(41); source <= 46; source++ {
|
||||
movers, count := 3, 12
|
||||
if source == 46 {
|
||||
movers, count = 73, 437
|
||||
}
|
||||
p := seat.Pipeline{V: 1, Pipeline: fmt.Sprintf("main.%d", source), Branch: "main", Source: source, Revision: source,
|
||||
Outcome: seat.PipelineOutcomePassed, Started: &at, Ended: &at, EndCount: 1}
|
||||
for i := range movers {
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: fmt.Sprintf("kept-%02d", i), Commit: "c0", Fingerprint: fmt.Sprintf("src2:k%02d", i)})
|
||||
}
|
||||
for i := range count {
|
||||
p.Steps = append(p.Steps, seat.StepResult{Module: p.Movers[i%movers].Module, Step: seat.StepResultStepBuild,
|
||||
Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomePassed, Started: &at, Ended: &at})
|
||||
}
|
||||
raw, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, p.Pipeline, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,6 +66,8 @@ type MoverStore struct {
|
||||
Raw RawState
|
||||
Logf func(string, ...any)
|
||||
|
||||
// mu guards the three maps, which every pipeline's reads and writes share across goroutines (H §774): the lock table,
|
||||
// the last reads and the records said inline. A pipeline's own lock is held across its read or write.
|
||||
mu sync.Mutex
|
||||
locks map[string]*sync.Mutex
|
||||
read map[string]readAt
|
||||
@@ -93,6 +95,42 @@ func (s *MoverStore) lock(key string) func() {
|
||||
return l.Unlock
|
||||
}
|
||||
|
||||
// seen is the last read of pipeline key, under mu.
|
||||
func (s *MoverStore) seen(key string) (readAt, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
r, ok := s.read[key]
|
||||
return r, ok
|
||||
}
|
||||
|
||||
// setSeen keeps the last read of pipeline key, under mu.
|
||||
func (s *MoverStore) setSeen(key string, r readAt) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.read[key] = r
|
||||
}
|
||||
|
||||
// markInline says whether pipeline key is said inline for the first time, marking it, under mu.
|
||||
func (s *MoverStore) markInline(key string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
first := !s.inline[key]
|
||||
s.inline[key] = true
|
||||
return first
|
||||
}
|
||||
|
||||
// forget drops pipeline key's last read, and its inline mark when inline is true, under mu.
|
||||
func (s *MoverStore) forget(key string, read, inline bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if read {
|
||||
delete(s.read, key)
|
||||
}
|
||||
if inline {
|
||||
delete(s.inline, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MoverStore) logf(f string, a ...any) {
|
||||
if s.Logf != nil {
|
||||
s.Logf(f, a...)
|
||||
@@ -233,10 +271,9 @@ func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) {
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, 0, err
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: revs}
|
||||
s.setSeen(key, readAt{rev: rev, movers: revs})
|
||||
if len(p.Steps) > 0 {
|
||||
if !s.inline[key] {
|
||||
s.inline[key] = true
|
||||
if s.markInline(key) {
|
||||
s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps))
|
||||
}
|
||||
return p, rev, nil
|
||||
@@ -271,7 +308,7 @@ func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint6
|
||||
return 0, err
|
||||
}
|
||||
// A mover key written since the read is another write between: the read named its revision.
|
||||
if seen, ok := s.read[key]; ok && seen.rev == revision {
|
||||
if seen, ok := s.seen(key); ok && seen.rev == revision {
|
||||
for k, r := range revs {
|
||||
if seen.movers[k] != r {
|
||||
return 0, ErrChanged
|
||||
@@ -301,9 +338,9 @@ func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint6
|
||||
}
|
||||
} else {
|
||||
s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want))
|
||||
delete(s.inline, key)
|
||||
s.forget(key, false, true)
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: after}
|
||||
s.setSeen(key, readAt{rev: rev, movers: after})
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
@@ -367,7 +404,7 @@ func (s *MoverStore) Create(key string, value []byte) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: after}
|
||||
s.setSeen(key, readAt{rev: rev, movers: after})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -386,8 +423,7 @@ func (s *MoverStore) Delete(key string) error {
|
||||
if err == nil {
|
||||
err = s.Raw.Delete(key)
|
||||
}
|
||||
delete(s.read, key)
|
||||
delete(s.inline, key)
|
||||
s.forget(key, true, true)
|
||||
unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user