mesh-controller/internal/chain/backend.go

package chain

import (
	"context"
	"errors"
	"sort"
	"sync"
	"time"
)

// Entry is one key's value at a revision. Caught is set on the entry a watch sends once, after the values the bucket
// held when it began, and carries no key.
type Entry struct {
	Key string
	// Deleted marks the end of a key (Remove); it holds no value.
	Deleted  bool
	Value    []byte
	Revision uint64
	At       time.Time
	Caught   bool
}

// Backend is one bucket of the chain. **It has no delete and no purge**: no key of the chain is ever removed (ADR
// 0408 §4), so nothing that holds a Backend can remove one.
type Backend interface {
	// Get is one key's value and revision; false when it holds none.
	Get(ctx context.Context, key string) (Entry, bool, error)
	// Create writes a key that holds nothing and fails with ErrExists when it holds something.
	Create(ctx context.Context, key string, value []byte) (uint64, error)
	// Update writes a key at the revision it was read at and fails with ErrMoved when it moved.
	Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error)
	// Keys is every key. An error is an error: never an empty bucket.
	Keys(ctx context.Context) ([]string, error)
	// Watch sends every key's current value, then a Caught entry, then each later write, until the context ends.
	Watch(ctx context.Context) (<-chan Entry, error)
	// History is a key's earlier values and its current one, oldest first, at most the bucket keeps.
	History(ctx context.Context, key string) ([]Entry, error)
	// Remove ends a key at the revision it was read at, with a marker: **its earlier values stay**, read back by History, and
	// nothing is purged. Used by an unassignment and by nothing else, in the delivery that moves the assignments; the
	// delivery after it keeps an unassigned key with an end mark and removes this (issue 1096).
	Remove(ctx context.Context, key string, revision uint64) error
}

var (
	// ErrMoved is a compare-and-set that lost: somebody wrote the key since it was read.
	ErrMoved = errors.New("the key was written by somebody else since it was read")
	// ErrExists is a create of a key that holds a value.
	ErrExists = errors.New("the key already holds a value")
	// ErrNotHeld is a write refused because this controller does not hold the lease.
	ErrNotHeld = errors.New("this controller does not hold the lease, so it writes nothing of the chain")
	// ErrTooLarge is a value over the largest the bus holds: never cut (ADR 0408 §6), so refused.
	ErrTooLarge = errors.New("the value is larger than the bus holds, and a value of the chain is never cut")
)

// Memory is a Backend held in this process, for tests and for nothing else. It keeps what the bucket keeps: every
// earlier value, up to History of them, and a revision that rises with every write of the bucket.
type Memory struct {
	// Keeps is how many values of a key it keeps; zero keeps sixty-four.
	Keeps int

	mu       sync.Mutex
	values   map[string][]Entry
	revision uint64
	watchers []*watcher
	// Now is the clock the entries are dated by; nil is time.Now.
	Now func() time.Time
	// writes counts the creates and updates taken.
	writes int
	// fail is what every call answers, when set.
	fail error
}

// NewMemory is an empty bucket.
func NewMemory() *Memory { return &Memory{values: map[string][]Entry{}} }

// SetFail makes every call answer err from now on, or none when err is nil: a bus that is away.
func (m *Memory) SetFail(err error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.fail = err
}

// Writes is how many writes the bucket took.
func (m *Memory) Writes() int {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.writes
}

func (m *Memory) keep() int {
	if m.Keeps > 0 {
		return m.Keeps
	}
	return 64
}

func (m *Memory) now() time.Time {
	if m.Now != nil {
		return m.Now()
	}
	return time.Now()
}

// last is a key's newest entry, false when it holds none or was removed.
func (m *Memory) last(key string) (Entry, bool) {
	h := m.values[key]
	if len(h) == 0 || h[len(h)-1].Deleted {
		return Entry{}, false
	}
	return h[len(h)-1], true
}

func (m *Memory) Get(_ context.Context, key string) (Entry, bool, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.fail != nil {
		return Entry{}, false, m.fail
	}
	e, ok := m.last(key)
	if !ok {
		return Entry{}, false, nil
	}
	return clone(e), true, nil
}

func (m *Memory) Remove(_ context.Context, key string, revision uint64) error {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.fail != nil {
		return m.fail
	}
	e, ok := m.last(key)
	if !ok || e.Revision != revision {
		return ErrMoved
	}
	m.revision++
	m.writes++
	h := append(m.values[key], Entry{Key: key, Deleted: true, Revision: m.revision, At: m.now()})
	if len(h) > m.keep() {
		h = h[len(h)-m.keep():]
	}
	m.values[key] = h
	return nil
}

func (m *Memory) Create(_ context.Context, key string, value []byte) (uint64, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.fail != nil {
		return 0, m.fail
	}
	if _, held := m.last(key); held {
		return 0, ErrExists
	}
	return m.write(key, value), nil
}

func (m *Memory) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.fail != nil {
		return 0, m.fail
	}
	e, held := m.last(key)
	if !held || e.Revision != revision {
		return 0, ErrMoved
	}
	return m.write(key, value), nil
}

// write is called with the lock held.
func (m *Memory) write(key string, value []byte) uint64 {
	m.revision++
	m.writes++
	e := Entry{Key: key, Value: append([]byte(nil), value...), Revision: m.revision, At: m.now()}
	h := append(m.values[key], e)
	if len(h) > m.keep() {
		h = h[len(h)-m.keep():]
	}
	m.values[key] = h
	for _, w := range m.watchers {
		w.push(clone(e))
	}
	return e.Revision
}

func (m *Memory) Keys(_ context.Context) ([]string, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.fail != nil {
		return nil, m.fail
	}
	out := make([]string, 0, len(m.values))
	for k := range m.values {
		if _, held := m.last(k); held {
			out = append(out, k)
		}
	}
	sort.Strings(out)
	return out, nil
}

// watcher is one watch of a Memory: an unbounded queue, so a watch that is slow is behind and never misses a write.
type watcher struct {
	mu   sync.Mutex
	q    []Entry
	wake chan struct{}
}

func (w *watcher) push(e Entry) {
	w.mu.Lock()
	w.q = append(w.q, e)
	w.mu.Unlock()
	select {
	case w.wake <- struct{}{}:
	default:
	}
}

func (w *watcher) pop() (Entry, bool) {
	w.mu.Lock()
	defer w.mu.Unlock()
	if len(w.q) == 0 {
		return Entry{}, false
	}
	e := w.q[0]
	w.q = w.q[1:]
	return e, true
}

func (m *Memory) Watch(ctx context.Context) (<-chan Entry, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.fail != nil {
		return nil, m.fail
	}
	w := &watcher{wake: make(chan struct{}, 1)}
	keys := make([]string, 0, len(m.values))
	for k := range m.values {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		if e, held := m.last(k); held {
			w.q = append(w.q, clone(e))
		}
	}
	w.q = append(w.q, Entry{Caught: true})
	m.watchers = append(m.watchers, w)
	out := make(chan Entry)
	go func() {
		defer close(out)
		defer func() {
			m.mu.Lock()
			for i, x := range m.watchers {
				if x == w {
					m.watchers = append(m.watchers[:i], m.watchers[i+1:]...)
					break
				}
			}
			m.mu.Unlock()
		}()
		for {
			e, ok := w.pop()
			if !ok {
				select {
				case <-ctx.Done():
					return
				case <-w.wake:
				}
				continue
			}
			select {
			case out <- e:
			case <-ctx.Done():
				return
			}
		}
	}()
	return out, nil
}

func (m *Memory) History(_ context.Context, key string) ([]Entry, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.fail != nil {
		return nil, m.fail
	}
	out := make([]Entry, 0, len(m.values[key]))
	for _, e := range m.values[key] {
		if !e.Deleted {
			out = append(out, clone(e))
		}
	}
	return out, nil
}

func clone(e Entry) Entry {
	e.Value = append([]byte(nil), e.Value...)
	return e
}