mesh-controller/internal/inventory/order.go

package inventory

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/jackc/pgx/v5"
)

// Order and one writer, as this context keeps them (novox/hq to-be 45 §6, Phase 2): the epochs the
// mesh issued, the epoch each machine was sent, whether its node-engine reads one, and the order of
// the account kept for it.

// ActsUnder gives this inventory the gate every write that acts passes (a plan's): the epoch of the
// controller lease this process acts under, or why it may not act. Nil — a test, a command reading —
// writes with no epoch and refuses nothing.
func (i *Inventory) ActsUnder(epoch func(ctx context.Context) (uint64, error)) { i.acting = epoch }

// actingEpoch is the epoch a write carries, nil when there is no gate or it claims none.
func (i *Inventory) actingEpoch(ctx context.Context) (*int64, error) {
	if i.acting == nil {
		return nil, nil
	}
	epoch, err := i.acting(ctx)
	if err != nil {
		return nil, err
	}
	if epoch == 0 {
		return nil, nil
	}
	e := int64(epoch)
	return &e, nil
}

// Epoch is one epoch the mesh issued.
type Epoch struct {
	Epoch    uint64
	Instance string
	Host     string
	Build    string
	Taken    time.Time
	// Ended is when it ended, nil while it is held; How is how: EpochReleased, EpochLost, EpochExpired.
	Ended *time.Time
	How   string
}

// How an epoch ends.
const (
	// EpochReleased is a holder that gave the lease back.
	EpochReleased = "released"
	// EpochLost is a holder whose own renewal was refused or failed, and which said so.
	EpochLost = "lost"
	// EpochExpired is a holder the next one found gone: it stopped renewing without saying anything.
	EpochExpired = "expired"
)

// HighestEpoch is the highest epoch the mesh has issued, zero before the first: the floor no new one
// may be at or under.
func (i *Inventory) HighestEpoch(ctx context.Context) (uint64, error) {
	var highest int64
	if err := i.store.Pool().QueryRow(ctx, `select coalesce(max(epoch), 0) from controller_epoch`).Scan(&highest); err != nil {
		return 0, fmt.Errorf("reading the highest epoch issued: %w", err)
	}
	return uint64(highest), nil
}

// TookEpoch records an epoch taken, and ends every earlier one still open as found expired: the lease
// was free to take, so whoever held it last neither holds it nor said it let go. givenBack is the bus's word
// that the key was given back (the lease's own delete marker, novox/hq issue 906): then the newest earlier
// epoch still open is ended as released — its holder gave the key back and had not yet recorded it — and only
// the ones before it as expired. Answers the epochs it ended as expired, newest first.
func (i *Inventory) TookEpoch(ctx context.Context, e Epoch, givenBack bool) ([]Epoch, error) {
	tx, err := i.store.Pool().Begin(ctx)
	if err != nil {
		return nil, err
	}
	defer func() { _ = tx.Rollback(ctx) }()
	if givenBack {
		if _, err := tx.Exec(ctx,
			`update controller_epoch set ended = now(), how = $2
			  where epoch = (select max(epoch) from controller_epoch where ended is null and epoch < $1)`,
			int64(e.Epoch), EpochReleased); err != nil {
			return nil, err
		}
	}
	rows, err := tx.Query(ctx,
		`update controller_epoch set ended = now(), how = $2
		  where ended is null and epoch < $1
		  returning epoch, instance, host, build, taken, ended, how`, int64(e.Epoch), EpochExpired)
	if err != nil {
		return nil, err
	}
	expired, err := scanEpochs(rows)
	if err != nil {
		return nil, err
	}
	if _, err := tx.Exec(ctx,
		`insert into controller_epoch (epoch, instance, host, build, taken) values ($1, $2, $3, $4, $5)
		 on conflict (epoch) do nothing`,
		int64(e.Epoch), e.Instance, e.Host, e.Build, e.Taken); err != nil {
		return nil, err
	}
	return expired, tx.Commit(ctx)
}

// EndEpoch records how an epoch ended. One already ended keeps its first word.
func (i *Inventory) EndEpoch(ctx context.Context, epoch uint64, how string) error {
	_, err := i.store.Pool().Exec(ctx,
		`update controller_epoch set ended = now(), how = $2 where epoch = $1 and ended is null`, int64(epoch), how)
	return err
}

// EpochOf is the epoch issued at a number; false when the mesh issued none there.
func (i *Inventory) EpochOf(ctx context.Context, epoch uint64) (Epoch, bool, error) {
	rows, err := i.store.Pool().Query(ctx,
		`select epoch, instance, host, build, taken, ended, how from controller_epoch where epoch = $1`, int64(epoch))
	if err != nil {
		return Epoch{}, false, err
	}
	found, err := scanEpochs(rows)
	if err != nil || len(found) == 0 {
		return Epoch{}, false, err
	}
	return found[0], true, nil
}

// EpochsSince is every epoch taken or ended since a moment, newest first.
func (i *Inventory) EpochsSince(ctx context.Context, since time.Time) ([]Epoch, error) {
	rows, err := i.store.Pool().Query(ctx,
		`select epoch, instance, host, build, taken, ended, how from controller_epoch
		  where taken >= $1 or ended >= $1 or ended is null order by epoch desc`, since)
	if err != nil {
		return nil, err
	}
	return scanEpochs(rows)
}

func scanEpochs(rows pgx.Rows) ([]Epoch, error) {
	defer rows.Close()
	var out []Epoch
	for rows.Next() {
		var e Epoch
		var epoch int64
		if err := rows.Scan(&epoch, &e.Instance, &e.Host, &e.Build, &e.Taken, &e.Ended, &e.How); err != nil {
			return nil, err
		}
		e.Epoch = uint64(epoch)
		out = append(out, e)
	}
	return out, rows.Err()
}

// ReportOrder is the order of an account: the declaration's epoch and sequence it is about, and the
// node-engine's own report sequence. Zero in any claims none.
type ReportOrder struct {
	Epoch          int64
	Sequence       int64
	ReportSequence int64
}

// ErrOlderAccount is an account refused because the one kept is newer.
var ErrOlderAccount = errors.New("an older account than the one kept")

// KeptOrder is the order of the account kept for a machine, by its id; zero when it carried none.
func (i *Inventory) KeptOrder(ctx context.Context, id string) (ReportOrder, error) {
	o, err := keptOrder(ctx, i.store.Pool(), id, "")
	if errors.Is(err, pgx.ErrNoRows) {
		return ReportOrder{}, nil
	}
	return o, err
}

func keptOrder(ctx context.Context, q queries, id, lock string) (ReportOrder, error) {
	var epoch, seq, rseq *int64
	err := q.QueryRow(ctx,
		`select reported_epoch, reported_sequence, report_sequence from node_report where node = $1`+lock, id).
		Scan(&epoch, &seq, &rseq)
	if err != nil {
		return ReportOrder{}, err
	}
	var o ReportOrder
	for _, f := range []struct {
		from *int64
		to   *int64
	}{{epoch, &o.Epoch}, {seq, &o.Sequence}, {rseq, &o.ReportSequence}} {
		if f.from != nil {
			*f.to = *f.from
		}
	}
	return o, nil
}

// RecordOrderedDoing is RecordDoing for an account that carries its order (novox/hq to-be 45 §6):
// written only when it is not older than the account kept, as olderThan judges — the rule is the
// link's (link.Account), stated once — and in one transaction with the row locked, so two accounts
// arriving together cannot both win. ErrOlderAccount when it is older; nothing is written then.
func (i *Inventory) RecordOrderedDoing(ctx context.Context, id string, d Doing, o ReportOrder,
	olderThan func(kept ReportOrder) bool) (news bool, err error) {
	tx, err := i.store.Pool().Begin(ctx)
	if err != nil {
		return false, err
	}
	defer func() { _ = tx.Rollback(ctx) }()
	kept, err := keptOrder(ctx, tx, id, " for update")
	switch {
	case errors.Is(err, pgx.ErrNoRows):
	case err != nil:
		return false, err
	case olderThan(kept):
		return false, ErrOlderAccount
	}
	news, err = recordDoing(ctx, tx, id, d)
	if err != nil {
		return false, err
	}
	if _, err := tx.Exec(ctx,
		`update node_report set reported_epoch = $2, reported_sequence = $3, report_sequence = $4 where node = $1`,
		id, nullIfZero(o.Epoch), nullIfZero(o.Sequence), nullIfZero(o.ReportSequence)); err != nil {
		return false, err
	}
	return news, tx.Commit(ctx)
}

// nullIfZero is a claimed order or none.
func nullIfZero(n int64) *int64 {
	if n == 0 {
		return nil
	}
	return &n
}