Every method of builds, build requests, rollouts, merges and assignments reads and writes the chain alone: the else branches that read the tables, the movers and MoveChain with their tests, chain_moved reads, ChainReads and the flags that switched a kind, the store's assignments mirror (KeepAssignmentPlans needs none of it: it uses only the AssignmentMirrored type, mirrors.go, and AssignmentValues, provisioning.go) and its tick, the move code of the controller's start, and ForTest's chain_moved row. The listed store tests of the move go with their functions; the tests of the readers, requests, assignments and plans that lived beside them stay in chain_store_test.go. Tests that replayed old data migrations over the build, assignment and pin tables are removed with the tables. The static test that no plan is written but through SavePlansMoving now reads the plan's chain path.
940 lines
27 KiB
Go
940 lines
27 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/chain"
|
|
)
|
|
|
|
// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan
|
|
// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk
|
|
// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside.
|
|
//
|
|
// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value,
|
|
// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no
|
|
// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the
|
|
// merge operations alone; a save keeps what the record holds.
|
|
|
|
// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the
|
|
// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by
|
|
// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one.
|
|
type rolloutView struct {
|
|
chain *chain.Chain
|
|
mu sync.RWMutex
|
|
entries map[string]viewEntry
|
|
ready chan struct{}
|
|
once sync.Once
|
|
err error
|
|
dead atomic.Bool
|
|
cancel context.CancelFunc
|
|
}
|
|
|
|
// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out.
|
|
type viewEntry struct {
|
|
raw []byte
|
|
rev uint64
|
|
sum planSummary
|
|
}
|
|
|
|
// planSummary is the part of a plan the surveys filter and sort by.
|
|
type planSummary struct {
|
|
ID string `json:"id"`
|
|
State string `json:"state"`
|
|
Created time.Time `json:"created"`
|
|
Repository string `json:"repository"`
|
|
Branch string `json:"branch"`
|
|
Merged time.Time `json:"merged"`
|
|
Release json.RawMessage `json:"release"`
|
|
Held []chain.HeldMerge `json:"held"`
|
|
}
|
|
|
|
func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling }
|
|
|
|
func (s planSummary) isRelease() bool {
|
|
r := strings.TrimSpace(string(s.Release))
|
|
return r != "" && r != "null"
|
|
}
|
|
|
|
func newRolloutView(c *chain.Chain) *rolloutView {
|
|
return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})}
|
|
}
|
|
|
|
func (v *rolloutView) start() {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
v.cancel = cancel
|
|
go func() {
|
|
w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx)
|
|
if err != nil {
|
|
v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err))
|
|
return
|
|
}
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
v.fail(ctx.Err())
|
|
return
|
|
case e, open := <-w:
|
|
if !open {
|
|
v.fail(errors.New("the rollouts bucket stopped answering"))
|
|
return
|
|
}
|
|
if e.Caught {
|
|
v.once.Do(func() { close(v.ready) })
|
|
continue
|
|
}
|
|
v.apply(e.Key, e.Value, e.Revision)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read.
|
|
func (v *rolloutView) fail(err error) {
|
|
v.mu.Lock()
|
|
if v.err == nil {
|
|
v.err = err
|
|
}
|
|
v.mu.Unlock()
|
|
v.dead.Store(true)
|
|
v.once.Do(func() { close(v.ready) })
|
|
}
|
|
|
|
func (v *rolloutView) stop() {
|
|
if v.cancel != nil {
|
|
v.cancel()
|
|
}
|
|
}
|
|
|
|
// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one.
|
|
func (v *rolloutView) apply(key string, raw []byte, rev uint64) {
|
|
var sum planSummary
|
|
if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" {
|
|
sum = planSummary{ID: key}
|
|
}
|
|
v.mu.Lock()
|
|
defer v.mu.Unlock()
|
|
if old, has := v.entries[key]; has && old.rev >= rev {
|
|
return
|
|
}
|
|
v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum}
|
|
}
|
|
|
|
func (v *rolloutView) list() []viewEntry {
|
|
v.mu.RLock()
|
|
defer v.mu.RUnlock()
|
|
out := make([]viewEntry, 0, len(v.entries))
|
|
for _, e := range v.entries {
|
|
out = append(out, e)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
|
|
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
|
|
i.state.mu.Lock()
|
|
v := i.state.view
|
|
if v == nil || v.chain != c || v.dead.Load() {
|
|
if v != nil {
|
|
v.stop()
|
|
}
|
|
v = newRolloutView(c)
|
|
i.state.view = v
|
|
v.start()
|
|
}
|
|
i.state.mu.Unlock()
|
|
select {
|
|
case <-v.ready:
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
v.mu.RLock()
|
|
defer v.mu.RUnlock()
|
|
if v.err != nil && len(v.entries) == 0 {
|
|
return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err)
|
|
}
|
|
return v, nil
|
|
}
|
|
|
|
// stopViews ends the view's watch: the inventory is closing.
|
|
func (i *Inventory) stopViews() {
|
|
i.state.mu.Lock()
|
|
defer i.state.mu.Unlock()
|
|
if i.state.view != nil {
|
|
i.state.view.stop()
|
|
i.state.view = nil
|
|
}
|
|
}
|
|
|
|
func normalizePlan(p *Plan) {
|
|
if p.Modules == nil {
|
|
p.Modules = map[string]*PlanModule{}
|
|
}
|
|
if p.TierEntered.IsZero() {
|
|
p.TierEntered = p.Created
|
|
}
|
|
if !p.Merged.IsZero() {
|
|
p.Merged = p.Merged.UTC()
|
|
}
|
|
}
|
|
|
|
func planOf(e viewEntry) (Plan, error) {
|
|
var p Plan
|
|
if err := json.Unmarshal(e.raw, &p); err != nil {
|
|
return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err)
|
|
}
|
|
normalizePlan(&p)
|
|
return p, nil
|
|
}
|
|
|
|
func sortCreated(entries []viewEntry) {
|
|
sort.SliceStable(entries, func(a, b int) bool {
|
|
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
|
|
return entries[a].sum.Created.Before(entries[b].sum.Created)
|
|
}
|
|
return entries[a].sum.ID < entries[b].sum.ID
|
|
})
|
|
}
|
|
|
|
func sortCreatedNewest(entries []viewEntry) {
|
|
sort.SliceStable(entries, func(a, b int) bool {
|
|
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
|
|
return entries[a].sum.Created.After(entries[b].sum.Created)
|
|
}
|
|
return entries[a].sum.ID > entries[b].sum.ID
|
|
})
|
|
}
|
|
|
|
// chainPlans is the plans that pass keep, in an order, read from the view.
|
|
func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) {
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var chosen []viewEntry
|
|
for _, e := range v.list() {
|
|
if e.sum.ID != WaitingRecord && keep(e) {
|
|
chosen = append(chosen, e)
|
|
}
|
|
}
|
|
order(chosen)
|
|
out := make([]Plan, 0, len(chosen))
|
|
for _, e := range chosen {
|
|
p, err := planOf(e)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, p)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) {
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return Plan{}, false, err
|
|
}
|
|
var chosen []viewEntry
|
|
for _, e := range v.list() {
|
|
if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() {
|
|
chosen = append(chosen, e)
|
|
}
|
|
}
|
|
sort.SliceStable(chosen, func(a, b int) bool {
|
|
if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) {
|
|
return chosen[a].sum.Merged.After(chosen[b].sum.Merged)
|
|
}
|
|
return chosen[a].sum.Created.After(chosen[b].sum.Created)
|
|
})
|
|
if len(chosen) == 0 {
|
|
return Plan{}, false, nil
|
|
}
|
|
p, err := planOf(chosen[0])
|
|
return p, err == nil, err
|
|
}
|
|
|
|
// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view
|
|
// learns of it.
|
|
func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) {
|
|
if !chain.ValidKey(id) {
|
|
// No plan has an id the bus takes no spelling of; it is a plain id no plan has.
|
|
return Plan{}, fmt.Errorf("no plan %s", id)
|
|
}
|
|
e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id)
|
|
if err != nil {
|
|
return Plan{}, err
|
|
}
|
|
if !found {
|
|
return Plan{}, fmt.Errorf("no plan %s", id)
|
|
}
|
|
if v, err := i.rolloutView(ctx, c); err == nil {
|
|
v.apply(id, e.Value, e.Revision)
|
|
}
|
|
return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}})
|
|
}
|
|
|
|
// ---- the merges
|
|
|
|
// mergeHolder is a record's copy of a merge.
|
|
type mergeHolder struct {
|
|
plan string
|
|
m chain.HeldMerge
|
|
rev uint64
|
|
}
|
|
|
|
func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit }
|
|
|
|
// mergeHolders is every record's copy of each merge, by the merge's key.
|
|
func mergeHolders(entries []viewEntry) map[string][]mergeHolder {
|
|
out := map[string][]mergeHolder{}
|
|
for _, e := range entries {
|
|
for _, m := range e.sum.Held {
|
|
k := mergeKey(m.Repository, m.Commit)
|
|
out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two
|
|
// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers
|
|
// before one no plan does, then the record written last.
|
|
func winner(copies []mergeHolder) mergeHolder {
|
|
best := copies[0]
|
|
for _, c := range copies[1:] {
|
|
switch {
|
|
case c.m.Placed.After(best.m.Placed):
|
|
best = c
|
|
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting:
|
|
best = c
|
|
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev:
|
|
best = c
|
|
}
|
|
}
|
|
return best
|
|
}
|
|
|
|
func winners(entries []viewEntry) map[string]mergeHolder {
|
|
out := map[string]mergeHolder{}
|
|
for k, copies := range mergeHolders(entries) {
|
|
out[k] = winner(copies)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func batchedOf(h mergeHolder) BatchedMerge {
|
|
b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit,
|
|
Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone}
|
|
if !h.m.Merged.IsZero() {
|
|
b.Merged = h.m.Merged.UTC()
|
|
}
|
|
if h.m.Waiting {
|
|
b.Plan = ""
|
|
}
|
|
return b
|
|
}
|
|
|
|
func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge {
|
|
h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(),
|
|
Event: m.Event, Alone: m.Alone, Placed: now.UTC()}
|
|
if !m.Merged.IsZero() {
|
|
h.Merged = m.Merged.UTC()
|
|
}
|
|
return h
|
|
}
|
|
|
|
func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) {
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return winners(v.list()), nil
|
|
}
|
|
|
|
func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return BatchedMerge{}, false, err
|
|
}
|
|
h, ok := w[mergeKey(repository, commit)]
|
|
if !ok {
|
|
return BatchedMerge{}, false, nil
|
|
}
|
|
return batchedOf(h), true, nil
|
|
}
|
|
|
|
// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard.
|
|
func sortMerges(ms []BatchedMerge, newest bool) {
|
|
sort.SliceStable(ms, func(a, b int) bool {
|
|
x, y := ms[a], ms[b]
|
|
switch {
|
|
case x.Merged.IsZero() != y.Merged.IsZero():
|
|
return !x.Merged.IsZero()
|
|
case !x.Merged.Equal(y.Merged):
|
|
if newest {
|
|
return x.Merged.After(y.Merged)
|
|
}
|
|
return x.Merged.Before(y.Merged)
|
|
}
|
|
if newest {
|
|
return x.Heard.After(y.Heard)
|
|
}
|
|
return x.Heard.Before(y.Heard)
|
|
})
|
|
}
|
|
|
|
func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []BatchedMerge
|
|
for _, h := range w {
|
|
if h.plan == plan && !h.m.Waiting {
|
|
out = append(out, batchedOf(h))
|
|
}
|
|
}
|
|
sortMerges(out, false)
|
|
return out, nil
|
|
}
|
|
|
|
func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []BatchedMerge
|
|
for _, h := range w {
|
|
if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) {
|
|
out = append(out, batchedOf(h))
|
|
}
|
|
}
|
|
sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) })
|
|
return out, nil
|
|
}
|
|
|
|
func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return false, time.Time{}, err
|
|
}
|
|
var first time.Time
|
|
for _, h := range w {
|
|
if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) {
|
|
first = h.m.Heard
|
|
}
|
|
}
|
|
if first.IsZero() {
|
|
return false, time.Time{}, nil
|
|
}
|
|
return true, first.UTC(), nil
|
|
}
|
|
|
|
func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []BatchedMerge
|
|
for _, h := range w {
|
|
if h.m.Waiting && h.m.Alone {
|
|
out = append(out, batchedOf(h))
|
|
}
|
|
}
|
|
sortMerges(out, true)
|
|
return out, nil
|
|
}
|
|
|
|
// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it.
|
|
type pendingMerge struct {
|
|
plan string
|
|
m chain.HeldMerge
|
|
}
|
|
|
|
// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has
|
|
// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is
|
|
// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3).
|
|
func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) {
|
|
waiting := m.Plan == ""
|
|
if waiting {
|
|
// A merge no plan answers is held, flagged, by the record kept for those.
|
|
m.Plan = WaitingRecord
|
|
if err := i.ensureWaitingRecord(ctx, c); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
key := mergeKey(m.Repository, m.Commit)
|
|
if _, known := winners(v.list())[key]; known {
|
|
return false, nil
|
|
}
|
|
held := heldOf(m, time.Now())
|
|
held.Waiting = waiting
|
|
found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
|
for _, x := range h {
|
|
if mergeKey(x.Repository, x.Commit) == key {
|
|
return h, false
|
|
}
|
|
}
|
|
return append(h, held), true
|
|
})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !found {
|
|
i.state.mu.Lock()
|
|
if i.state.pending == nil {
|
|
i.state.pending = map[string]pendingMerge{}
|
|
}
|
|
i.state.pending[key] = pendingMerge{plan: m.Plan, m: held}
|
|
i.state.mu.Unlock()
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store
|
|
// kept as a row with a null plan. It is no plan: the surveys of plans leave it out.
|
|
const WaitingRecord = "waiting-merges"
|
|
|
|
// ensureWaitingRecord makes the record of the merges no plan answers, once.
|
|
func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error {
|
|
b := chain.RolloutsOf[Plan](c)
|
|
if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found {
|
|
return err
|
|
}
|
|
body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
|
|
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rev, err := b.CreateRaw(ctx, WaitingRecord, body)
|
|
if errors.Is(err, chain.ErrExists) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if v, verr := i.rolloutView(ctx, c); verr == nil {
|
|
v.apply(WaitingRecord, body, rev)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it.
|
|
var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved")
|
|
|
|
// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again
|
|
// when it moved. It answers false when the plan has no record.
|
|
func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) {
|
|
b := chain.RolloutsOf[Plan](c)
|
|
for try := 0; try < casTries; try++ {
|
|
e, found, err := b.GetRaw(ctx, id)
|
|
if err != nil || !found {
|
|
return false, err
|
|
}
|
|
var record map[string]json.RawMessage
|
|
if err := json.Unmarshal(e.Value, &record); err != nil {
|
|
return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err)
|
|
}
|
|
var held []chain.HeldMerge
|
|
if raw, has := record["held"]; has {
|
|
if err := json.Unmarshal(raw, &held); err != nil {
|
|
return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err)
|
|
}
|
|
}
|
|
next, changed := edit(held)
|
|
if !changed {
|
|
return true, nil
|
|
}
|
|
if len(next) == 0 {
|
|
delete(record, "held")
|
|
} else {
|
|
raw, err := json.Marshal(next)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
record["held"] = raw
|
|
}
|
|
body, err := json.Marshal(record)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
rev, err := b.UpdateRaw(ctx, id, body, e.Revision)
|
|
if errors.Is(err, chain.ErrMoved) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err)
|
|
}
|
|
if v, verr := i.rolloutView(ctx, c); verr == nil {
|
|
v.apply(id, body, rev)
|
|
}
|
|
return true, nil
|
|
}
|
|
return true, fmt.Errorf("the merges of plan %s kept moving under the write", id)
|
|
}
|
|
|
|
// recordEdit is what the merges of one record gain, lose and have flagged in one act.
|
|
type recordEdit struct {
|
|
add []chain.HeldMerge
|
|
drop map[string]bool
|
|
flag map[string]chain.HeldMerge
|
|
}
|
|
|
|
func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge {
|
|
out := make([]chain.HeldMerge, 0, len(held)+len(e.add))
|
|
for _, h := range held {
|
|
k := mergeKey(h.Repository, h.Commit)
|
|
if e.drop[k] {
|
|
continue
|
|
}
|
|
if f, has := e.flag[k]; has {
|
|
h.Waiting, h.Alone = f.Waiting, f.Alone
|
|
}
|
|
out = append(out, h)
|
|
}
|
|
for _, a := range e.add {
|
|
k := mergeKey(a.Repository, a.Commit)
|
|
dup := false
|
|
for _, h := range out {
|
|
if mergeKey(h.Repository, h.Commit) == k {
|
|
dup = true
|
|
break
|
|
}
|
|
}
|
|
if !dup {
|
|
out = append(out, a)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 }
|
|
|
|
// ---- the save
|
|
|
|
// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save
|
|
// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one.
|
|
func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) {
|
|
if old == nil {
|
|
return now, nil // a new plan enters its first tier now
|
|
}
|
|
if old.State == PlanAssembling || old.State == PlanQueued {
|
|
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
|
|
}
|
|
since := old.TierEntered
|
|
if since.IsZero() {
|
|
since = old.Created
|
|
}
|
|
wasOpen := old.State == PlanBuilding || old.State == PlanRolling
|
|
if !wasOpen || (old.Tier == p.Tier && p.Open()) {
|
|
return since, nil // still in the tier, or already ended
|
|
}
|
|
var modules []string
|
|
if old.Tier >= 0 && old.Tier < len(p.Tiers) {
|
|
modules = p.Tiers[old.Tier]
|
|
}
|
|
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State)
|
|
if p.Open() {
|
|
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier)
|
|
}
|
|
return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier),
|
|
Started: since, Took: now.Sub(since), Detail: detail}
|
|
}
|
|
|
|
// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by
|
|
// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is
|
|
// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a
|
|
// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092).
|
|
func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) {
|
|
epoch, err := i.actingEpoch(ctx)
|
|
if err != nil {
|
|
if len(plans) > 0 {
|
|
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
|
|
}
|
|
return err
|
|
}
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b := chain.RolloutsOf[Plan](c)
|
|
marked := time.Now()
|
|
undo := make([]func(), 0, len(plans))
|
|
defer func() {
|
|
if err != nil {
|
|
for _, u := range undo {
|
|
u()
|
|
}
|
|
}
|
|
}()
|
|
for _, p := range plans {
|
|
undo = append(undo, p.markWalk(marked))
|
|
}
|
|
|
|
// Every plan still at the revision it was read at, before anything is written.
|
|
cur := make(map[string]chain.Kept[Plan], len(plans))
|
|
for _, p := range plans {
|
|
k, found, err := b.Get(ctx, p.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found {
|
|
if k.Value.Revision != p.Revision {
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
|
}
|
|
cur[p.ID] = k
|
|
}
|
|
}
|
|
|
|
// What each record gains, loses and has flagged.
|
|
now := time.Now().UTC()
|
|
edits := map[string]*recordEdit{}
|
|
editOf := func(id string) *recordEdit {
|
|
if edits[id] == nil {
|
|
edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}}
|
|
}
|
|
return edits[id]
|
|
}
|
|
copies := mergeHolders(v.list())
|
|
for _, m := range moves {
|
|
key := mergeKey(m.Repository, m.Commit)
|
|
held, has := copies[key]
|
|
if !has {
|
|
// A merge carried for a plan not saved yet goes where it is moved.
|
|
i.state.mu.Lock()
|
|
if pm, pending := i.state.pending[key]; pending && m.Plan != "" {
|
|
pm.plan, pm.m.Alone = m.Plan, m.Alone
|
|
i.state.pending[key] = pm
|
|
}
|
|
i.state.mu.Unlock()
|
|
continue
|
|
}
|
|
w := winner(held)
|
|
if m.Plan == "" {
|
|
f := w.m
|
|
f.Waiting, f.Alone = true, m.Alone
|
|
editOf(w.plan).flag[key] = f
|
|
continue
|
|
}
|
|
if w.plan == m.Plan {
|
|
f := w.m
|
|
f.Waiting, f.Alone = false, m.Alone
|
|
editOf(w.plan).flag[key] = f
|
|
continue
|
|
}
|
|
moved := w.m
|
|
moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now
|
|
editOf(m.Plan).add = append(editOf(m.Plan).add, moved)
|
|
for _, h := range held {
|
|
editOf(h.plan).drop[key] = true
|
|
}
|
|
}
|
|
|
|
// The records not among the plans: receivers now, givers after.
|
|
for id, e := range edits {
|
|
if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 {
|
|
continue
|
|
}
|
|
e := e
|
|
found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
|
return (&recordEdit{add: e.add}).apply(h), true
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
// Nothing is taken from the record that holds a merge to give it to a plan that has none.
|
|
return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add))
|
|
}
|
|
}
|
|
|
|
// The plans, those that gain a merge first.
|
|
order := append([]*Plan(nil), plans...)
|
|
sort.SliceStable(order, func(a, b int) bool {
|
|
x, y := edits[order[a].ID], edits[order[b].ID]
|
|
return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0)
|
|
})
|
|
type written struct {
|
|
revision int64
|
|
entered time.Time
|
|
held []chain.HeldMerge
|
|
duration *Duration
|
|
}
|
|
done := map[string]written{}
|
|
for _, p := range order {
|
|
var w written
|
|
for try := 0; ; try++ {
|
|
if try == casTries {
|
|
return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID)
|
|
}
|
|
k, found := cur[p.ID]
|
|
if try > 0 {
|
|
fresh, still, gerr := b.Get(ctx, p.ID)
|
|
if gerr != nil {
|
|
return gerr
|
|
}
|
|
if still && fresh.Value.Revision != p.Revision {
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
|
}
|
|
k, found = fresh, still
|
|
cur[p.ID] = k
|
|
}
|
|
record := *p
|
|
var old *Plan
|
|
var held []chain.HeldMerge
|
|
if found {
|
|
o := k.Value
|
|
old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...)
|
|
record.Revision = k.Value.Revision + 1
|
|
} else {
|
|
record.Revision = 1
|
|
}
|
|
if e := edits[p.ID]; e != nil {
|
|
held = e.apply(held)
|
|
}
|
|
i.state.mu.Lock()
|
|
for key, pm := range i.state.pending {
|
|
if pm.plan == p.ID {
|
|
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
|
|
_ = key
|
|
}
|
|
}
|
|
i.state.mu.Unlock()
|
|
record.Held = held
|
|
entered, duration := tierLeftOf(old, *p, time.Now())
|
|
record.TierEntered, record.Updated = entered, time.Now().UTC()
|
|
if epoch != nil {
|
|
record.Epoch = uint64(*epoch)
|
|
} else {
|
|
record.Epoch = 0
|
|
}
|
|
body, merr := json.Marshal(record)
|
|
if merr != nil {
|
|
return merr
|
|
}
|
|
var rev uint64
|
|
var werr error
|
|
if found {
|
|
rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision)
|
|
} else {
|
|
rev, werr = createRaw(ctx, c, p.ID, body)
|
|
}
|
|
if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) {
|
|
if !found {
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
|
}
|
|
continue
|
|
}
|
|
if werr != nil {
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr)
|
|
}
|
|
v.apply(p.ID, body, rev)
|
|
w = written{revision: record.Revision, entered: entered, held: held, duration: duration}
|
|
break
|
|
}
|
|
done[p.ID] = w
|
|
}
|
|
|
|
// The records that gave a merge up, now it is held by the one it went to.
|
|
for id, e := range edits {
|
|
if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) {
|
|
continue
|
|
}
|
|
e := e
|
|
if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
|
return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
i.state.mu.Lock()
|
|
for key, pm := range i.state.pending {
|
|
if _, saved := done[pm.plan]; saved {
|
|
delete(i.state.pending, key)
|
|
}
|
|
}
|
|
i.state.mu.Unlock()
|
|
for _, p := range plans {
|
|
w := done[p.ID]
|
|
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
|
|
if epoch != nil {
|
|
p.Epoch = uint64(*epoch)
|
|
} else {
|
|
p.Epoch = 0
|
|
}
|
|
if w.duration != nil {
|
|
// After the write: the key of the duration is the plan and its tier, so a measurement made twice is one.
|
|
record := i.RecordDuration
|
|
if i.recorded != nil {
|
|
record = i.recorded
|
|
}
|
|
if derr := record(ctx, *w.duration); derr != nil {
|
|
fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr)
|
|
}
|
|
}
|
|
if PlanSaved != nil {
|
|
PlanSaved(i, *p)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func planNamed(plans []*Plan, id string) bool {
|
|
for _, p := range plans {
|
|
if p.ID == id {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// createRaw writes a plan's first record.
|
|
func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) {
|
|
return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body)
|
|
}
|
|
|
|
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
|
|
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
|
|
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
|
|
c, err := i.theChain(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
repaired := 0
|
|
for key, copies := range mergeHolders(v.list()) {
|
|
if len(copies) < 2 {
|
|
continue
|
|
}
|
|
w := winner(copies)
|
|
for _, h := range copies {
|
|
if h.plan == w.plan {
|
|
continue
|
|
}
|
|
key := key
|
|
if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
|
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
|
|
}); err != nil {
|
|
return repaired, err
|
|
}
|
|
repaired++
|
|
}
|
|
}
|
|
return repaired, nil
|
|
}
|