mesh-controller/internal/inventory/chain.go

package inventory

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"sync/atomic"

	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgconn"

	"github.com/novox/mesh-controller/internal/chain"
)

// **The chain moves to the bus kind by kind** (novox/hq ADR 0408 §8, issue 981). The tables of builds, rollouts and
// assignments are copied once into the chain's buckets by the serving controller at its start (MoveChain), which then
// records each kind it moved in chain_moved (migration 0104). **Until a kind is recorded there nothing reads its bucket;
// once it is, nothing reads its tables**: ChainReads answers which, and every method of a kind asks it first.

// The kinds of the chain, as chain_moved names them.
const (
	KindBuilds   = "builds"
	KindRollouts = "rollouts"
)

// testChains are the chains in memory a test made for its store, by the store's address: every inventory opened on that
// store is given the same chain, as every process of a mesh is on the same bus. Empty outside a test.
var testChains sync.Map

// ErrNoChain is a kind moved to the bus read by an inventory that has no chain attached: a process that did not
// open the bus. Refused loudly, never answered from the tables the move left behind.
var ErrNoChain = errors.New("this kind of record lives on the bus, and this process has not opened the bus")

// movedKinds is which kinds this process has read as moved: once moved, always moved, so it is read from the store
// until it is true and not after.
type movedKinds struct {
	mu   sync.Mutex
	kind map[string]bool
	open ChainOpener
	// view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that
	// is not saved yet.
	view    *rolloutView
	pending map[string]pendingMerge
	// switching is set when the assignments begin to be switched to the bus and kept once they are: the store's mirror of
	// them writes nothing from then on (AssignmentsMirrorMayWrite). Cleared only if the switch fails.
	switching atomic.Bool
}

// AssignmentsMirrorMayWrite says the store's mirror of the assignments may still write the bucket: the assignments are
// neither being switched to the bus nor on it. Asked before every write of the mirror's writer, not once a pass.
func (i *Inventory) AssignmentsMirrorMayWrite() bool {
	i.moved.mu.Lock()
	moved := i.moved.kind[KindAssignments]
	i.moved.mu.Unlock()
	return !moved && !i.moved.switching.Load()
}

// ChainOpener opens the chain on the bus when a process first needs it: a command that reads and never moved anything has
// no bus connection until a kind it reads turns out to be moved.
type ChainOpener func(ctx context.Context) (*chain.Chain, error)

// UseChainOpener gives the inventory the means to open the chain when it needs it and none is attached.
func (i *Inventory) UseChainOpener(open ChainOpener) {
	i.moved.mu.Lock()
	i.moved.open = open
	i.moved.mu.Unlock()
}

// UseChain attaches the chain the moved kinds are read and written through. The serving controller attaches the bus's,
// holding the lease for its writes; a test attaches one in memory.
func (i *Inventory) UseChain(c *chain.Chain) {
	i.moved.mu.Lock()
	defer i.moved.mu.Unlock()
	i.chain = c
	if i.moved.view != nil {
		i.moved.view.stop()
		i.moved.view = nil
	}
}

// attached is the chain this inventory has, nil when none.
func (i *Inventory) attached() *chain.Chain {
	i.moved.mu.Lock()
	defer i.moved.mu.Unlock()
	return i.chain
}

// ChainReads says whether a kind of the chain is read from its bucket: chain_moved names it. It answers the chain to read
// through, and ErrNoChain when the kind is moved and none is attached.
func (i *Inventory) ChainReads(ctx context.Context, kind string) (*chain.Chain, bool, error) {
	moved, err := i.kindMoved(ctx, kind)
	if err != nil || !moved {
		return nil, false, err
	}
	if kind == KindBuilds {
		// **The builds are read under their modules** (issue 1121): until the layout is made, a read would find nothing.
		laid, err := i.kindMoved(ctx, KindBuildsLayout)
		if err != nil {
			return nil, false, err
		}
		if !laid {
			return nil, false, fmt.Errorf("%w: the builds are not laid out under their modules yet; the move at the controller's start does it", ErrNoChain)
		}
	}
	i.moved.mu.Lock()
	defer i.moved.mu.Unlock()
	if i.chain == nil && i.moved.open != nil {
		c, err := i.moved.open(ctx)
		if err != nil {
			return nil, false, fmt.Errorf("%w: %s: %v", ErrNoChain, kind, err)
		}
		i.chain = c
	}
	if i.chain == nil {
		return nil, false, fmt.Errorf("%w: %s", ErrNoChain, kind)
	}
	return i.chain, true, nil
}

// kindMoved is whether chain_moved names a kind; once it does, always, so the store is asked until it is true.
func (i *Inventory) kindMoved(ctx context.Context, kind string) (bool, error) {
	i.moved.mu.Lock()
	known := i.moved.kind[kind]
	i.moved.mu.Unlock()
	if known {
		return true, nil
	}
	var moved bool
	err := i.store.Pool().QueryRow(ctx, `select counts ? $1 from chain_moved`, kind).Scan(&moved)
	var pgErr *pgconn.PgError
	if errors.Is(err, pgx.ErrNoRows) || (errors.As(err, &pgErr) && pgErr.Code == "42P01") {
		// No row, or a store older than the migration that makes the table: nothing moved.
		return false, nil
	}
	if err != nil {
		return false, fmt.Errorf("whether %s moved to the bus cannot be read: %w", kind, err)
	}
	if !moved {
		return false, nil
	}
	i.moved.mu.Lock()
	if i.moved.kind == nil {
		i.moved.kind = map[string]bool{}
	}
	i.moved.kind[kind] = true
	i.moved.mu.Unlock()
	return true, nil
}

// ChainMoved is, per kind moved, how many records the move wrote; empty before any is.
func (i *Inventory) ChainMoved(ctx context.Context) (map[string]int, error) {
	var counts map[string]int
	err := i.store.Pool().QueryRow(ctx, `select counts from chain_moved`).Scan(&counts)
	if errors.Is(err, pgx.ErrNoRows) {
		return map[string]int{}, nil
	}
	return counts, err
}

// NewForTest is an inventory with no store, for a test of what needs none.
func NewForTest() *Inventory { return &Inventory{moved: &movedKinds{}} }

// BeginSwitchForTest sets the flag a kind's switch sets, as a switch under way does.
func (i *Inventory) BeginSwitchForTest(kind string) {
	if kind == KindAssignments {
		i.moved.switching.Store(true)
	}
}