Keep rollouts and the merges they answer in the rollouts bucket (issue 1092)

Once chain_moved records the kind, a plan is a key of the rollouts bucket under its id and the open batch is the same key in state assembling. A save refuses a copy another save moved since (the plan's own Revision, kept in its value, counts saves) and keeps the merges the record holds, so a copy read before a merge moved cannot lay it back. The merges a batch or walk answers are the record's held field, with the columns the table had; a merge no plan answers stays, flagged, in the record that held it last, or in the record kept for those. A merge moved to a plan is written to the receiving record first and then taken from the giving one; one found in both is read from the one it entered last and repaired at the next cut. Plans and merges are read from a view kept by one watch of the bucket. A bus that does not answer, a lease lost and a key moved are now not-now like the store being away, so a merge whose plan could not be written is held and handed over again, never acknowledged. The start moves release_plan and batched_merge into the bucket, one kind, counted against both tables.

internal/chain/backend.go

@@ -55,7 +55,7 @@ type Memory struct {
 	mu       sync.Mutex
 	values   map[string][]Entry
 	revision uint64
-	watchers []chan Entry
+	watchers []*watcher
 	// Now is the clock the entries are dated by; nil is time.Now.
 	Now func() time.Time
 	// writes counts the creates and updates taken.
@@ -144,10 +144,7 @@ func (m *Memory) write(key string, value []byte) uint64 {
 	}
 	m.values[key] = h
 	for _, w := range m.watchers {
-		select {
-		case w <- clone(e):
-		default:
-		}
+		w.push(clone(e))
 	}
 	return e.Revision
 }
@@ -166,14 +163,41 @@ func (m *Memory) Keys(_ context.Context) ([]string, error) {
 	return out, nil
 }
 
+// watcher is one watch of a Memory: an unbounded queue, so a watch that is slow is behind and never misses a write.
+type watcher struct {
+	mu   sync.Mutex
+	q    []Entry
+	wake chan struct{}
+}
+
+func (w *watcher) push(e Entry) {
+	w.mu.Lock()
+	w.q = append(w.q, e)
+	w.mu.Unlock()
+	select {
+	case w.wake <- struct{}{}:
+	default:
+	}
+}
+
+func (w *watcher) pop() (Entry, bool) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if len(w.q) == 0 {
+		return Entry{}, false
+	}
+	e := w.q[0]
+	w.q = w.q[1:]
+	return e, true
+}
+
 func (m *Memory) Watch(ctx context.Context) (<-chan Entry, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if m.fail != nil {
 		return nil, m.fail
 	}
-	// Room for the current values, the caught mark, and a burst of later writes.
-	w := make(chan Entry, len(m.values)+1+1024)
+	w := &watcher{wake: make(chan struct{}, 1)}
 	keys := make([]string, 0, len(m.values))
 	for k := range m.values {
 		keys = append(keys, k)
@@ -181,30 +205,37 @@ func (m *Memory) Watch(ctx context.Context) (<-chan Entry, error) {
 	sort.Strings(keys)
 	for _, k := range keys {
 		h := m.values[k]
-		w <- clone(h[len(h)-1])
+		w.q = append(w.q, clone(h[len(h)-1]))
 	}
-	w <- Entry{Caught: true}
+	w.q = append(w.q, Entry{Caught: true})
 	m.watchers = append(m.watchers, w)
 	out := make(chan Entry)
 	go func() {
 		defer close(out)
+		defer func() {
+			m.mu.Lock()
+			for i, x := range m.watchers {
+				if x == w {
+					m.watchers = append(m.watchers[:i], m.watchers[i+1:]...)
+					break
+				}
+			}
+			m.mu.Unlock()
+		}()
 		for {
-			select {
-			case <-ctx.Done():
-				m.mu.Lock()
-				for i, x := range m.watchers {
-					if x == w {
-						m.watchers = append(m.watchers[:i], m.watchers[i+1:]...)
-						break
-					}
-				}
-				m.mu.Unlock()
-				return
-			case e := <-w:
+			e, ok := w.pop()
+			if !ok {
 				select {
-				case out <- e:
 				case <-ctx.Done():
+					return
+				case <-w.wake:
 				}
+				continue
+			}
+			select {
+			case out <- e:
+			case <-ctx.Done():
+				return
 			}
 		}
 	}()