Fold a pending assignment into the value of its assignment key (issue 1094)
A wait for a build is the value of <node>.<module> with state pending and a pending object (id p-<unix nanoseconds>, a string), so every reader that skips a value with a state skips it, and the controller's own lists skip it through Stands. The transitions are compare-and-set on the key's revision (claim, release, settle, repoint, mark); assigning over a wait carries it into the value that stands, applied when it was waiting; a plan pass never overwrites it; one open wait per machine and module, and a key that holds the module refuses a new one. The table is no longer read or written, ForgetEndedPending is gone (no key is removed), and a wait of a machine the store dropped is none. Callers change type only; their conditions are the next issue.
This commit is contained in:
+219
-122
@@ -3,7 +3,9 @@ package inventory
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -214,17 +216,19 @@ func (i *Inventory) requests(ctx context.Context) ([]RequestOutcome, error) {
|
||||
|
||||
// The states of a pending assignment.
|
||||
const (
|
||||
PendingWaiting = "waiting"
|
||||
PendingApplying = "applying"
|
||||
PendingApplied = "applied"
|
||||
PendingRefused = "refused"
|
||||
PendingExpired = "expired"
|
||||
PendingWithdrawn = "withdrawn"
|
||||
PendingWaiting = chain.PendingWaiting
|
||||
PendingApplying = chain.PendingApplying
|
||||
PendingApplied = chain.PendingApplied
|
||||
PendingRefused = chain.PendingRefused
|
||||
PendingExpired = chain.PendingExpired
|
||||
PendingWithdrawn = chain.PendingWithdrawn
|
||||
)
|
||||
|
||||
// PendingAssignment is an assignment kept until its module is registered (ADR 0261).
|
||||
// PendingAssignment is an assignment kept until its module is registered (ADR 0261): the value of its key in the
|
||||
// assignments bucket while it waits (state pending), and the record carried into the value that stands once it is made.
|
||||
type PendingAssignment struct {
|
||||
ID int64
|
||||
// ID is p-<unix nanoseconds>, made when it was recorded.
|
||||
ID string
|
||||
Node string
|
||||
Module string
|
||||
// Build is the build it waits for; Repository and Path where that build is from.
|
||||
@@ -247,195 +251,288 @@ func (p PendingAssignment) Open() bool {
|
||||
// ErrAlreadyPending is a second pending assignment of one module to one machine.
|
||||
var ErrAlreadyPending = errors.New("that assignment is already waiting for its build")
|
||||
|
||||
const pendingColumns = `p.id, n.name, p.module, p.build_id, p.repository, p.source_path, p.since, p.state, p.note,
|
||||
p.settled_at, p.raised_at, p.acknowledged_at, p.cleared_at`
|
||||
// ErrOnTheMachine is a pending assignment of a module that is on the machine already.
|
||||
var ErrOnTheMachine = errors.New("that module is on the machine already")
|
||||
|
||||
func scanPending(rows pgx.Rows) ([]PendingAssignment, error) {
|
||||
defer rows.Close()
|
||||
var out []PendingAssignment
|
||||
for rows.Next() {
|
||||
var p PendingAssignment
|
||||
if err := rows.Scan(&p.ID, &p.Node, &p.Module, &p.Build, &p.Repository, &p.Path, &p.Since, &p.State,
|
||||
&p.Note, &p.Settled, &p.Raised, &p.Acknowledged, &p.Cleared); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
func pendingOf(v chain.Assignment) PendingAssignment {
|
||||
p := v.Pending
|
||||
return PendingAssignment{ID: p.ID, Node: v.Node, Module: v.Module, Build: p.Build, Repository: p.Repository, Path: p.Path,
|
||||
Since: p.Since, State: p.State, Note: p.Note, Settled: p.Settled, Raised: p.Raised, Acknowledged: p.Acknowledged,
|
||||
Cleared: p.Cleared}
|
||||
}
|
||||
|
||||
// RecordPending keeps an assignment until its module is registered. One is open per machine and module.
|
||||
// pendingValues is every value of the bucket that holds a pending assignment, on a machine the store knows (a removed
|
||||
// machine's is none, as the node's foreign key made it), with its key and revision.
|
||||
func (i *Inventory) pendingValues(ctx context.Context) ([]chain.Kept[chain.Assignment], error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the pending assignments cannot be read from the bus: %w", err)
|
||||
}
|
||||
nodes, err := i.nodeNames(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []chain.Kept[chain.Assignment]
|
||||
for key, k := range all {
|
||||
if k.Value.Pending != nil && nodes[k.Value.Node] {
|
||||
k.Key = key
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool {
|
||||
x, y := out[a].Value.Pending, out[b].Value.Pending
|
||||
if !x.Since.Equal(y.Since) {
|
||||
return x.Since.Before(y.Since)
|
||||
}
|
||||
return x.ID < y.ID
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func pendingList(kept []chain.Kept[chain.Assignment], keep func(chain.Pending) bool) []PendingAssignment {
|
||||
var out []PendingAssignment
|
||||
for _, k := range kept {
|
||||
if keep(*k.Value.Pending) {
|
||||
out = append(out, pendingOf(k.Value))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changePending changes the pending assignment of an id by compare-and-set on the key that holds it, read again when it
|
||||
// moved: false when no key holds that id now, or change says no.
|
||||
func (i *Inventory) changePending(ctx context.Context, id string, change func(v *chain.Assignment) bool) (bool, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for try := 0; try < casTries; try++ {
|
||||
held, err := i.pendingValues(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var at *chain.Kept[chain.Assignment]
|
||||
for k := range held {
|
||||
if held[k].Value.Pending.ID == id {
|
||||
at = &held[k]
|
||||
}
|
||||
}
|
||||
if at == nil {
|
||||
return false, nil
|
||||
}
|
||||
v := at.Value
|
||||
p := *v.Pending
|
||||
v.Pending = &p
|
||||
if !change(&v) {
|
||||
return false, nil
|
||||
}
|
||||
if _, err := c.Assignments.Update(ctx, at.Key, v, at.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the pending assignment %s kept moving under the write", id)
|
||||
}
|
||||
|
||||
// RecordPending keeps an assignment until its module is registered, as the value of its key (state pending). One is open per
|
||||
// machine and module; a key that holds the module on the machine refuses it, and one that holds an ended assignment or a
|
||||
// settled wait is written over.
|
||||
func (i *Inventory) RecordPending(ctx context.Context, p PendingAssignment) (PendingAssignment, error) {
|
||||
node, err := i.NodeByName(ctx, p.Node)
|
||||
if err != nil {
|
||||
return PendingAssignment{}, err
|
||||
}
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`insert into pending_assignment (node, module, build_id, repository, source_path)
|
||||
values ($1, $2, $3, $4, $5)
|
||||
on conflict (node, module) where state in ('waiting', 'applying') do nothing
|
||||
returning id, since`,
|
||||
node.ID, p.Module, p.Build, p.Repository, strings.Trim(p.Path, "/")).Scan(&p.ID, &p.Since)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return PendingAssignment{}, ErrAlreadyPending
|
||||
}
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return PendingAssignment{}, err
|
||||
}
|
||||
p.State = PendingWaiting
|
||||
return p, nil
|
||||
key := assignmentKey(node.Name, p.Module)
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, key)
|
||||
if err != nil {
|
||||
return PendingAssignment{}, err
|
||||
}
|
||||
switch {
|
||||
case found && cur.Value.Stands():
|
||||
return PendingAssignment{}, fmt.Errorf("%w: %s on %s", ErrOnTheMachine, p.Module, node.Name)
|
||||
case found && cur.Value.Pending != nil && cur.Value.State == chain.StatePending && cur.Value.Pending.Open():
|
||||
return PendingAssignment{}, ErrAlreadyPending
|
||||
}
|
||||
since := time.Now().UTC()
|
||||
p.ID, p.Since, p.State = fmt.Sprintf("p-%d", since.UnixNano()), since, PendingWaiting
|
||||
p.Path = strings.Trim(p.Path, "/")
|
||||
value := chain.Assignment{Node: node.Name, Module: p.Module, State: chain.StatePending, Pending: &chain.Pending{
|
||||
ID: p.ID, State: PendingWaiting, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: since}}
|
||||
if found {
|
||||
_, err = c.Assignments.Update(ctx, key, value, cur.Revision)
|
||||
} else {
|
||||
_, err = c.Assignments.Create(ctx, key, value)
|
||||
}
|
||||
if errors.Is(err, chain.ErrMoved) || errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return PendingAssignment{}, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
return PendingAssignment{}, fmt.Errorf("the assignment of %s to %s kept moving under the write", p.Module, node.Name)
|
||||
}
|
||||
|
||||
// RepointPending makes a waiting pending assignment wait for another build. False when it no longer waits.
|
||||
func (i *Inventory) RepointPending(ctx context.Context, id int64, build, repository, sourcePath string) (bool, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update pending_assignment set build_id = $2, repository = $3, source_path = $4
|
||||
where id = $1 and state = 'waiting'`, id, build, repository, strings.Trim(sourcePath, "/"))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
func (i *Inventory) RepointPending(ctx context.Context, id string, build, repository, sourcePath string) (bool, error) {
|
||||
return i.changePending(ctx, id, func(v *chain.Assignment) bool {
|
||||
if v.Pending.State != PendingWaiting {
|
||||
return false
|
||||
}
|
||||
v.Pending.Build, v.Pending.Repository, v.Pending.Path = build, repository, strings.Trim(sourcePath, "/")
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// Pending is every pending assignment still waiting, oldest first, when endedSince is zero. Otherwise it is
|
||||
// every open one and every one ended since then.
|
||||
func (i *Inventory) Pending(ctx context.Context, endedSince time.Time) ([]PendingAssignment, error) {
|
||||
if endedSince.IsZero() {
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
|
||||
from pending_assignment p join node n on n.id = p.node
|
||||
where p.state = 'waiting' order by p.since, p.id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanPending(rows)
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
|
||||
from pending_assignment p join node n on n.id = p.node
|
||||
where p.state in ('waiting', 'applying') or p.settled_at >= $1
|
||||
order by p.since, p.id`, endedSince)
|
||||
held, err := i.pendingValues(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanPending(rows)
|
||||
if endedSince.IsZero() {
|
||||
return pendingList(held, func(p chain.Pending) bool { return p.State == PendingWaiting }), nil
|
||||
}
|
||||
return pendingList(held, func(p chain.Pending) bool {
|
||||
return p.Open() || (p.Settled != nil && !p.Settled.Before(endedSince))
|
||||
}), nil
|
||||
}
|
||||
|
||||
// PendingFor is every pending assignment of one module to one machine, newest first.
|
||||
// PendingFor is the pending assignment of one module to one machine: the one its key holds now, which is the newest
|
||||
// (an earlier one is read from the key's history, which nothing here needs).
|
||||
func (i *Inventory) PendingFor(ctx context.Context, node, module string) ([]PendingAssignment, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
|
||||
from pending_assignment p join node n on n.id = p.node
|
||||
where n.name = $1 and p.module = $2 order by p.since desc, p.id desc`, node, module)
|
||||
held, err := i.pendingValues(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanPending(rows)
|
||||
var out []PendingAssignment
|
||||
for _, k := range held {
|
||||
if k.Value.Node == node && k.Value.Module == module {
|
||||
out = append(out, pendingOf(k.Value))
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Stuck is every pending assignment left applying since before the time given: a controller that claimed
|
||||
// it and stopped before it said what came of it.
|
||||
func (i *Inventory) Stuck(ctx context.Context, before time.Time) ([]PendingAssignment, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
|
||||
from pending_assignment p join node n on n.id = p.node
|
||||
where p.state = 'applying' and p.settled_at < $1 order by p.id`, before)
|
||||
held, err := i.pendingValues(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanPending(rows)
|
||||
return pendingList(held, func(p chain.Pending) bool {
|
||||
return p.State == PendingApplying && p.Settled != nil && p.Settled.Before(before)
|
||||
}), nil
|
||||
}
|
||||
|
||||
// ToRaise is every pending assignment that expired or was refused and has not been raised as a condition;
|
||||
// ToClear every one raised and not cleared.
|
||||
func (i *Inventory) ToRaise(ctx context.Context) ([]PendingAssignment, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
|
||||
from pending_assignment p join node n on n.id = p.node
|
||||
where p.state in ('expired', 'refused') and p.raised_at is null order by p.id`)
|
||||
held, err := i.pendingValues(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanPending(rows)
|
||||
return pendingList(held, func(p chain.Pending) bool {
|
||||
return (p.State == PendingExpired || p.State == PendingRefused) && p.Raised == nil
|
||||
}), nil
|
||||
}
|
||||
|
||||
func (i *Inventory) ToClear(ctx context.Context) ([]PendingAssignment, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+pendingColumns+`
|
||||
from pending_assignment p join node n on n.id = p.node
|
||||
where p.raised_at is not null and p.cleared_at is null order by p.id`)
|
||||
held, err := i.pendingValues(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanPending(rows)
|
||||
return pendingList(held, func(p chain.Pending) bool { return p.Raised != nil && p.Cleared == nil }), nil
|
||||
}
|
||||
|
||||
// MarkPending stamps a pending assignment raised, acknowledged or cleared, now.
|
||||
func (i *Inventory) MarkPending(ctx context.Context, id int64, what string) error {
|
||||
column := map[string]string{"raised": "raised_at", "acknowledged": "acknowledged_at", "cleared": "cleared_at"}[what]
|
||||
if column == "" {
|
||||
func (i *Inventory) MarkPending(ctx context.Context, id string, what string) error {
|
||||
if what != "raised" && what != "acknowledged" && what != "cleared" {
|
||||
return errors.New("a pending assignment is marked raised, acknowledged or cleared")
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update pending_assignment set `+column+` = coalesce(`+column+`, now()) where id = $1`, id)
|
||||
now := time.Now().UTC()
|
||||
_, err := i.changePending(ctx, id, func(v *chain.Assignment) bool {
|
||||
mark := map[string]**time.Time{"raised": &v.Pending.Raised, "acknowledged": &v.Pending.Acknowledged, "cleared": &v.Pending.Cleared}[what]
|
||||
if *mark != nil {
|
||||
return false // the first stamp stands
|
||||
}
|
||||
*mark = &now
|
||||
return true
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// ClaimPending takes a waiting pending assignment for making, so nothing else settles or withdraws it while
|
||||
// it is made. False when it no longer waits: another process took it, or it was withdrawn.
|
||||
func (i *Inventory) ClaimPending(ctx context.Context, id int64) (bool, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update pending_assignment set state = 'applying', settled_at = now() where id = $1 and state = 'waiting'`, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
func (i *Inventory) ClaimPending(ctx context.Context, id string) (bool, error) {
|
||||
now := time.Now().UTC()
|
||||
return i.changePending(ctx, id, func(v *chain.Assignment) bool {
|
||||
if !chain.CanMove(v.Pending.State, PendingApplying) {
|
||||
return false
|
||||
}
|
||||
v.Pending.State, v.Pending.Settled = PendingApplying, &now
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// ReleaseClaim puts a claimed pending assignment back to waiting.
|
||||
func (i *Inventory) ReleaseClaim(ctx context.Context, id int64) error {
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update pending_assignment set state = 'waiting', settled_at = null where id = $1 and state = 'applying'`, id)
|
||||
func (i *Inventory) ReleaseClaim(ctx context.Context, id string) error {
|
||||
_, err := i.changePending(ctx, id, func(v *chain.Assignment) bool {
|
||||
if v.Pending.State != PendingApplying {
|
||||
return false
|
||||
}
|
||||
v.Pending.State, v.Pending.Settled = PendingWaiting, nil
|
||||
return true
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// SettlePending ends a pending assignment in the state it is read in (from): waiting for an expiry or a
|
||||
// withdrawal, applying for the outcome of making it. False when it was no longer in that state.
|
||||
func (i *Inventory) SettlePending(ctx context.Context, id int64, from, state, note string) (bool, error) {
|
||||
func (i *Inventory) SettlePending(ctx context.Context, id string, from, state, note string) (bool, error) {
|
||||
if state == PendingWaiting || state == PendingApplying {
|
||||
return false, errors.New("a pending assignment is settled to applied, refused, expired or withdrawn")
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update pending_assignment set state = $3, note = $4, settled_at = now()
|
||||
where id = $1 and state = $2`, id, from, state, note)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
now := time.Now().UTC()
|
||||
return i.changePending(ctx, id, func(v *chain.Assignment) bool {
|
||||
if v.Pending.State != from || !chain.CanSettle(from, state) {
|
||||
return false
|
||||
}
|
||||
v.Pending.State, v.Pending.Note, v.Pending.Settled = state, note, &now
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// ForgetEndedPending deletes ended pending assignments settled more than KeptFor ago, and says how many. One
|
||||
// raised as a condition is kept until that condition is cleared: deleting it would leave the condition
|
||||
// with nothing to answer it.
|
||||
func (i *Inventory) ForgetEndedPending(ctx context.Context, now time.Time) (int64, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`delete from pending_assignment where state not in ('waiting', 'applying') and settled_at < $1
|
||||
and (raised_at is null or cleared_at is not null)`,
|
||||
now.Add(-KeptFor))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// PendingKnown says which of these pending assignments are still on record.
|
||||
func (i *Inventory) PendingKnown(ctx context.Context, ids []int64) (map[int64]bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select id from pending_assignment where id = any($1)`, ids)
|
||||
// PendingKnown says which of these pending assignments are still on record: a key holds them now. One a newer pending
|
||||
// assignment of its key replaced is not.
|
||||
func (i *Inventory) PendingKnown(ctx context.Context, ids []string) (map[string]bool, error) {
|
||||
held, err := i.pendingValues(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[int64]bool{}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[id] = true
|
||||
want := map[string]bool{}
|
||||
for _, id := range ids {
|
||||
want[id] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
out := map[string]bool{}
|
||||
for _, k := range held {
|
||||
if want[k.Value.Pending.ID] {
|
||||
out[k.Value.Pending.ID] = true
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user