Once chain_moved records the kind, a plan is a key of the rollouts bucket under its id and the open batch is the same key in state assembling. A save refuses a copy another save moved since (the plan's own Revision, kept in its value, counts saves) and keeps the merges the record holds, so a copy read before a merge moved cannot lay it back. The merges a batch or walk answers are the record's held field, with the columns the table had; a merge no plan answers stays, flagged, in the record that held it last, or in the record kept for those. A merge moved to a plan is written to the receiving record first and then taken from the giving one; one found in both is read from the one it entered last and repaired at the next cut. Plans and merges are read from a view kept by one watch of the bucket. A bus that does not answer, a lease lost and a key moved are now not-now like the store being away, so a merge whose plan could not be written is held and handed over again, never acknowledged. The start moves release_plan and batched_merge into the bucket, one kind, counted against both tables.
940 lines
27 KiB
Go
940 lines
27 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/chain"
|
|
)
|
|
|
|
// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan
|
|
// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk
|
|
// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside.
|
|
//
|
|
// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value,
|
|
// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no
|
|
// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the
|
|
// merge operations alone; a save keeps what the record holds.
|
|
|
|
// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the
|
|
// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by
|
|
// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one.
|
|
type rolloutView struct {
|
|
chain *chain.Chain
|
|
mu sync.RWMutex
|
|
entries map[string]viewEntry
|
|
ready chan struct{}
|
|
once sync.Once
|
|
err error
|
|
dead atomic.Bool
|
|
cancel context.CancelFunc
|
|
}
|
|
|
|
// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out.
|
|
type viewEntry struct {
|
|
raw []byte
|
|
rev uint64
|
|
sum planSummary
|
|
}
|
|
|
|
// planSummary is the part of a plan the surveys filter and sort by.
|
|
type planSummary struct {
|
|
ID string `json:"id"`
|
|
State string `json:"state"`
|
|
Created time.Time `json:"created"`
|
|
Repository string `json:"repository"`
|
|
Branch string `json:"branch"`
|
|
Merged time.Time `json:"merged"`
|
|
Release json.RawMessage `json:"release"`
|
|
Held []chain.HeldMerge `json:"held"`
|
|
}
|
|
|
|
func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling }
|
|
|
|
func (s planSummary) isRelease() bool {
|
|
r := strings.TrimSpace(string(s.Release))
|
|
return r != "" && r != "null"
|
|
}
|
|
|
|
func newRolloutView(c *chain.Chain) *rolloutView {
|
|
return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})}
|
|
}
|
|
|
|
func (v *rolloutView) start() {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
v.cancel = cancel
|
|
go func() {
|
|
w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx)
|
|
if err != nil {
|
|
v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err))
|
|
return
|
|
}
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
v.fail(ctx.Err())
|
|
return
|
|
case e, open := <-w:
|
|
if !open {
|
|
v.fail(errors.New("the rollouts bucket stopped answering"))
|
|
return
|
|
}
|
|
if e.Caught {
|
|
v.once.Do(func() { close(v.ready) })
|
|
continue
|
|
}
|
|
v.apply(e.Key, e.Value, e.Revision)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read.
|
|
func (v *rolloutView) fail(err error) {
|
|
v.mu.Lock()
|
|
if v.err == nil {
|
|
v.err = err
|
|
}
|
|
v.mu.Unlock()
|
|
v.dead.Store(true)
|
|
v.once.Do(func() { close(v.ready) })
|
|
}
|
|
|
|
func (v *rolloutView) stop() {
|
|
if v.cancel != nil {
|
|
v.cancel()
|
|
}
|
|
}
|
|
|
|
// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one.
|
|
func (v *rolloutView) apply(key string, raw []byte, rev uint64) {
|
|
var sum planSummary
|
|
if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" {
|
|
sum = planSummary{ID: key}
|
|
}
|
|
v.mu.Lock()
|
|
defer v.mu.Unlock()
|
|
if old, has := v.entries[key]; has && old.rev >= rev {
|
|
return
|
|
}
|
|
v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum}
|
|
}
|
|
|
|
func (v *rolloutView) list() []viewEntry {
|
|
v.mu.RLock()
|
|
defer v.mu.RUnlock()
|
|
out := make([]viewEntry, 0, len(v.entries))
|
|
for _, e := range v.entries {
|
|
out = append(out, e)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
|
|
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
|
|
i.moved.mu.Lock()
|
|
v := i.moved.view
|
|
if v == nil || v.chain != c || v.dead.Load() {
|
|
if v != nil {
|
|
v.stop()
|
|
}
|
|
v = newRolloutView(c)
|
|
i.moved.view = v
|
|
v.start()
|
|
}
|
|
i.moved.mu.Unlock()
|
|
select {
|
|
case <-v.ready:
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
v.mu.RLock()
|
|
defer v.mu.RUnlock()
|
|
if v.err != nil && len(v.entries) == 0 {
|
|
return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err)
|
|
}
|
|
return v, nil
|
|
}
|
|
|
|
// stopViews ends the view's watch: the inventory is closing.
|
|
func (i *Inventory) stopViews() {
|
|
i.moved.mu.Lock()
|
|
defer i.moved.mu.Unlock()
|
|
if i.moved.view != nil {
|
|
i.moved.view.stop()
|
|
i.moved.view = nil
|
|
}
|
|
}
|
|
|
|
func normalizePlan(p *Plan) {
|
|
if p.Modules == nil {
|
|
p.Modules = map[string]*PlanModule{}
|
|
}
|
|
if p.TierEntered.IsZero() {
|
|
p.TierEntered = p.Created
|
|
}
|
|
if !p.Merged.IsZero() {
|
|
p.Merged = p.Merged.UTC()
|
|
}
|
|
}
|
|
|
|
func planOf(e viewEntry) (Plan, error) {
|
|
var p Plan
|
|
if err := json.Unmarshal(e.raw, &p); err != nil {
|
|
return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err)
|
|
}
|
|
normalizePlan(&p)
|
|
return p, nil
|
|
}
|
|
|
|
func sortCreated(entries []viewEntry) {
|
|
sort.SliceStable(entries, func(a, b int) bool {
|
|
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
|
|
return entries[a].sum.Created.Before(entries[b].sum.Created)
|
|
}
|
|
return entries[a].sum.ID < entries[b].sum.ID
|
|
})
|
|
}
|
|
|
|
func sortCreatedNewest(entries []viewEntry) {
|
|
sort.SliceStable(entries, func(a, b int) bool {
|
|
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
|
|
return entries[a].sum.Created.After(entries[b].sum.Created)
|
|
}
|
|
return entries[a].sum.ID > entries[b].sum.ID
|
|
})
|
|
}
|
|
|
|
// chainPlans is the plans that pass keep, in an order, read from the view.
|
|
func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) {
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var chosen []viewEntry
|
|
for _, e := range v.list() {
|
|
if e.sum.ID != WaitingRecord && keep(e) {
|
|
chosen = append(chosen, e)
|
|
}
|
|
}
|
|
order(chosen)
|
|
out := make([]Plan, 0, len(chosen))
|
|
for _, e := range chosen {
|
|
p, err := planOf(e)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, p)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) {
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return Plan{}, false, err
|
|
}
|
|
var chosen []viewEntry
|
|
for _, e := range v.list() {
|
|
if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() {
|
|
chosen = append(chosen, e)
|
|
}
|
|
}
|
|
sort.SliceStable(chosen, func(a, b int) bool {
|
|
if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) {
|
|
return chosen[a].sum.Merged.After(chosen[b].sum.Merged)
|
|
}
|
|
return chosen[a].sum.Created.After(chosen[b].sum.Created)
|
|
})
|
|
if len(chosen) == 0 {
|
|
return Plan{}, false, nil
|
|
}
|
|
p, err := planOf(chosen[0])
|
|
return p, err == nil, err
|
|
}
|
|
|
|
// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view
|
|
// learns of it.
|
|
func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) {
|
|
if !chain.ValidKey(id) {
|
|
// No plan has an id the bus takes no spelling of; it is a plain id no plan has.
|
|
return Plan{}, fmt.Errorf("no plan %s", id)
|
|
}
|
|
e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id)
|
|
if err != nil {
|
|
return Plan{}, err
|
|
}
|
|
if !found {
|
|
return Plan{}, fmt.Errorf("no plan %s", id)
|
|
}
|
|
if v, err := i.rolloutView(ctx, c); err == nil {
|
|
v.apply(id, e.Value, e.Revision)
|
|
}
|
|
return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}})
|
|
}
|
|
|
|
// ---- the merges
|
|
|
|
// mergeHolder is a record's copy of a merge.
|
|
type mergeHolder struct {
|
|
plan string
|
|
m chain.HeldMerge
|
|
rev uint64
|
|
}
|
|
|
|
func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit }
|
|
|
|
// mergeHolders is every record's copy of each merge, by the merge's key.
|
|
func mergeHolders(entries []viewEntry) map[string][]mergeHolder {
|
|
out := map[string][]mergeHolder{}
|
|
for _, e := range entries {
|
|
for _, m := range e.sum.Held {
|
|
k := mergeKey(m.Repository, m.Commit)
|
|
out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two
|
|
// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers
|
|
// before one no plan does, then the record written last.
|
|
func winner(copies []mergeHolder) mergeHolder {
|
|
best := copies[0]
|
|
for _, c := range copies[1:] {
|
|
switch {
|
|
case c.m.Placed.After(best.m.Placed):
|
|
best = c
|
|
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting:
|
|
best = c
|
|
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev:
|
|
best = c
|
|
}
|
|
}
|
|
return best
|
|
}
|
|
|
|
func winners(entries []viewEntry) map[string]mergeHolder {
|
|
out := map[string]mergeHolder{}
|
|
for k, copies := range mergeHolders(entries) {
|
|
out[k] = winner(copies)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func batchedOf(h mergeHolder) BatchedMerge {
|
|
b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit,
|
|
Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone}
|
|
if !h.m.Merged.IsZero() {
|
|
b.Merged = h.m.Merged.UTC()
|
|
}
|
|
if h.m.Waiting {
|
|
b.Plan = ""
|
|
}
|
|
return b
|
|
}
|
|
|
|
func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge {
|
|
h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(),
|
|
Event: m.Event, Alone: m.Alone, Placed: now.UTC()}
|
|
if !m.Merged.IsZero() {
|
|
h.Merged = m.Merged.UTC()
|
|
}
|
|
return h
|
|
}
|
|
|
|
func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) {
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return winners(v.list()), nil
|
|
}
|
|
|
|
func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return BatchedMerge{}, false, err
|
|
}
|
|
h, ok := w[mergeKey(repository, commit)]
|
|
if !ok {
|
|
return BatchedMerge{}, false, nil
|
|
}
|
|
return batchedOf(h), true, nil
|
|
}
|
|
|
|
// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard.
|
|
func sortMerges(ms []BatchedMerge, newest bool) {
|
|
sort.SliceStable(ms, func(a, b int) bool {
|
|
x, y := ms[a], ms[b]
|
|
switch {
|
|
case x.Merged.IsZero() != y.Merged.IsZero():
|
|
return !x.Merged.IsZero()
|
|
case !x.Merged.Equal(y.Merged):
|
|
if newest {
|
|
return x.Merged.After(y.Merged)
|
|
}
|
|
return x.Merged.Before(y.Merged)
|
|
}
|
|
if newest {
|
|
return x.Heard.After(y.Heard)
|
|
}
|
|
return x.Heard.Before(y.Heard)
|
|
})
|
|
}
|
|
|
|
func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []BatchedMerge
|
|
for _, h := range w {
|
|
if h.plan == plan && !h.m.Waiting {
|
|
out = append(out, batchedOf(h))
|
|
}
|
|
}
|
|
sortMerges(out, false)
|
|
return out, nil
|
|
}
|
|
|
|
func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []BatchedMerge
|
|
for _, h := range w {
|
|
if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) {
|
|
out = append(out, batchedOf(h))
|
|
}
|
|
}
|
|
sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) })
|
|
return out, nil
|
|
}
|
|
|
|
func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return false, time.Time{}, err
|
|
}
|
|
var first time.Time
|
|
for _, h := range w {
|
|
if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) {
|
|
first = h.m.Heard
|
|
}
|
|
}
|
|
if first.IsZero() {
|
|
return false, time.Time{}, nil
|
|
}
|
|
return true, first.UTC(), nil
|
|
}
|
|
|
|
func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) {
|
|
w, err := i.chainWinners(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out []BatchedMerge
|
|
for _, h := range w {
|
|
if h.m.Waiting && h.m.Alone {
|
|
out = append(out, batchedOf(h))
|
|
}
|
|
}
|
|
sortMerges(out, true)
|
|
return out, nil
|
|
}
|
|
|
|
// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it.
|
|
type pendingMerge struct {
|
|
plan string
|
|
m chain.HeldMerge
|
|
}
|
|
|
|
// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has
|
|
// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is
|
|
// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3).
|
|
func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) {
|
|
waiting := m.Plan == ""
|
|
if waiting {
|
|
// A merge no plan answers is held, flagged, by the record kept for those.
|
|
m.Plan = WaitingRecord
|
|
if err := i.ensureWaitingRecord(ctx, c); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
key := mergeKey(m.Repository, m.Commit)
|
|
if _, known := winners(v.list())[key]; known {
|
|
return false, nil
|
|
}
|
|
held := heldOf(m, time.Now())
|
|
held.Waiting = waiting
|
|
found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
|
for _, x := range h {
|
|
if mergeKey(x.Repository, x.Commit) == key {
|
|
return h, false
|
|
}
|
|
}
|
|
return append(h, held), true
|
|
})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !found {
|
|
i.moved.mu.Lock()
|
|
if i.moved.pending == nil {
|
|
i.moved.pending = map[string]pendingMerge{}
|
|
}
|
|
i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held}
|
|
i.moved.mu.Unlock()
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store
|
|
// kept as a row with a null plan. It is no plan: the surveys of plans leave it out.
|
|
const WaitingRecord = "waiting-merges"
|
|
|
|
// ensureWaitingRecord makes the record of the merges no plan answers, once.
|
|
func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error {
|
|
b := chain.RolloutsOf[Plan](c)
|
|
if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found {
|
|
return err
|
|
}
|
|
body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
|
|
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rev, err := b.CreateRaw(ctx, WaitingRecord, body)
|
|
if errors.Is(err, chain.ErrExists) {
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if v, verr := i.rolloutView(ctx, c); verr == nil {
|
|
v.apply(WaitingRecord, body, rev)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it.
|
|
var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved")
|
|
|
|
// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again
|
|
// when it moved. It answers false when the plan has no record.
|
|
func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) {
|
|
b := chain.RolloutsOf[Plan](c)
|
|
for try := 0; try < casTries; try++ {
|
|
e, found, err := b.GetRaw(ctx, id)
|
|
if err != nil || !found {
|
|
return false, err
|
|
}
|
|
var record map[string]json.RawMessage
|
|
if err := json.Unmarshal(e.Value, &record); err != nil {
|
|
return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err)
|
|
}
|
|
var held []chain.HeldMerge
|
|
if raw, has := record["held"]; has {
|
|
if err := json.Unmarshal(raw, &held); err != nil {
|
|
return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err)
|
|
}
|
|
}
|
|
next, changed := edit(held)
|
|
if !changed {
|
|
return true, nil
|
|
}
|
|
if len(next) == 0 {
|
|
delete(record, "held")
|
|
} else {
|
|
raw, err := json.Marshal(next)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
record["held"] = raw
|
|
}
|
|
body, err := json.Marshal(record)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
rev, err := b.UpdateRaw(ctx, id, body, e.Revision)
|
|
if errors.Is(err, chain.ErrMoved) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err)
|
|
}
|
|
if v, verr := i.rolloutView(ctx, c); verr == nil {
|
|
v.apply(id, body, rev)
|
|
}
|
|
return true, nil
|
|
}
|
|
return true, fmt.Errorf("the merges of plan %s kept moving under the write", id)
|
|
}
|
|
|
|
// recordEdit is what the merges of one record gain, lose and have flagged in one act.
|
|
type recordEdit struct {
|
|
add []chain.HeldMerge
|
|
drop map[string]bool
|
|
flag map[string]chain.HeldMerge
|
|
}
|
|
|
|
func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge {
|
|
out := make([]chain.HeldMerge, 0, len(held)+len(e.add))
|
|
for _, h := range held {
|
|
k := mergeKey(h.Repository, h.Commit)
|
|
if e.drop[k] {
|
|
continue
|
|
}
|
|
if f, has := e.flag[k]; has {
|
|
h.Waiting, h.Alone = f.Waiting, f.Alone
|
|
}
|
|
out = append(out, h)
|
|
}
|
|
for _, a := range e.add {
|
|
k := mergeKey(a.Repository, a.Commit)
|
|
dup := false
|
|
for _, h := range out {
|
|
if mergeKey(h.Repository, h.Commit) == k {
|
|
dup = true
|
|
break
|
|
}
|
|
}
|
|
if !dup {
|
|
out = append(out, a)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 }
|
|
|
|
// ---- the save
|
|
|
|
// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save
|
|
// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one.
|
|
func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) {
|
|
if old == nil {
|
|
return now, nil // a new plan enters its first tier now
|
|
}
|
|
if old.State == PlanAssembling || old.State == PlanQueued {
|
|
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
|
|
}
|
|
since := old.TierEntered
|
|
if since.IsZero() {
|
|
since = old.Created
|
|
}
|
|
wasOpen := old.State == PlanBuilding || old.State == PlanRolling
|
|
if !wasOpen || (old.Tier == p.Tier && p.Open()) {
|
|
return since, nil // still in the tier, or already ended
|
|
}
|
|
var modules []string
|
|
if old.Tier >= 0 && old.Tier < len(p.Tiers) {
|
|
modules = p.Tiers[old.Tier]
|
|
}
|
|
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State)
|
|
if p.Open() {
|
|
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier)
|
|
}
|
|
return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier),
|
|
Started: since, Took: now.Sub(since), Detail: detail}
|
|
}
|
|
|
|
// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by
|
|
// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is
|
|
// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a
|
|
// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092).
|
|
func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) {
|
|
epoch, err := i.actingEpoch(ctx)
|
|
if err != nil {
|
|
if len(plans) > 0 {
|
|
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
|
|
}
|
|
return err
|
|
}
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
b := chain.RolloutsOf[Plan](c)
|
|
marked := time.Now()
|
|
undo := make([]func(), 0, len(plans))
|
|
defer func() {
|
|
if err != nil {
|
|
for _, u := range undo {
|
|
u()
|
|
}
|
|
}
|
|
}()
|
|
for _, p := range plans {
|
|
undo = append(undo, p.markWalk(marked))
|
|
}
|
|
|
|
// Every plan still at the revision it was read at, before anything is written.
|
|
cur := make(map[string]chain.Kept[Plan], len(plans))
|
|
for _, p := range plans {
|
|
k, found, err := b.Get(ctx, p.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if found {
|
|
if k.Value.Revision != p.Revision {
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
|
}
|
|
cur[p.ID] = k
|
|
}
|
|
}
|
|
|
|
// What each record gains, loses and has flagged.
|
|
now := time.Now().UTC()
|
|
edits := map[string]*recordEdit{}
|
|
editOf := func(id string) *recordEdit {
|
|
if edits[id] == nil {
|
|
edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}}
|
|
}
|
|
return edits[id]
|
|
}
|
|
copies := mergeHolders(v.list())
|
|
for _, m := range moves {
|
|
key := mergeKey(m.Repository, m.Commit)
|
|
held, has := copies[key]
|
|
if !has {
|
|
// A merge carried for a plan not saved yet goes where it is moved.
|
|
i.moved.mu.Lock()
|
|
if pm, pending := i.moved.pending[key]; pending && m.Plan != "" {
|
|
pm.plan, pm.m.Alone = m.Plan, m.Alone
|
|
i.moved.pending[key] = pm
|
|
}
|
|
i.moved.mu.Unlock()
|
|
continue
|
|
}
|
|
w := winner(held)
|
|
if m.Plan == "" {
|
|
f := w.m
|
|
f.Waiting, f.Alone = true, m.Alone
|
|
editOf(w.plan).flag[key] = f
|
|
continue
|
|
}
|
|
if w.plan == m.Plan {
|
|
f := w.m
|
|
f.Waiting, f.Alone = false, m.Alone
|
|
editOf(w.plan).flag[key] = f
|
|
continue
|
|
}
|
|
moved := w.m
|
|
moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now
|
|
editOf(m.Plan).add = append(editOf(m.Plan).add, moved)
|
|
for _, h := range held {
|
|
editOf(h.plan).drop[key] = true
|
|
}
|
|
}
|
|
|
|
// The records not among the plans: receivers now, givers after.
|
|
for id, e := range edits {
|
|
if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 {
|
|
continue
|
|
}
|
|
e := e
|
|
found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
|
return (&recordEdit{add: e.add}).apply(h), true
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
// Nothing is taken from the record that holds a merge to give it to a plan that has none.
|
|
return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add))
|
|
}
|
|
}
|
|
|
|
// The plans, those that gain a merge first.
|
|
order := append([]*Plan(nil), plans...)
|
|
sort.SliceStable(order, func(a, b int) bool {
|
|
x, y := edits[order[a].ID], edits[order[b].ID]
|
|
return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0)
|
|
})
|
|
type written struct {
|
|
revision int64
|
|
entered time.Time
|
|
held []chain.HeldMerge
|
|
duration *Duration
|
|
}
|
|
done := map[string]written{}
|
|
for _, p := range order {
|
|
var w written
|
|
for try := 0; ; try++ {
|
|
if try == casTries {
|
|
return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID)
|
|
}
|
|
k, found := cur[p.ID]
|
|
if try > 0 {
|
|
fresh, still, gerr := b.Get(ctx, p.ID)
|
|
if gerr != nil {
|
|
return gerr
|
|
}
|
|
if still && fresh.Value.Revision != p.Revision {
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
|
}
|
|
k, found = fresh, still
|
|
cur[p.ID] = k
|
|
}
|
|
record := *p
|
|
var old *Plan
|
|
var held []chain.HeldMerge
|
|
if found {
|
|
o := k.Value
|
|
old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...)
|
|
record.Revision = k.Value.Revision + 1
|
|
} else {
|
|
record.Revision = 1
|
|
}
|
|
if e := edits[p.ID]; e != nil {
|
|
held = e.apply(held)
|
|
}
|
|
i.moved.mu.Lock()
|
|
for key, pm := range i.moved.pending {
|
|
if pm.plan == p.ID {
|
|
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
|
|
_ = key
|
|
}
|
|
}
|
|
i.moved.mu.Unlock()
|
|
record.Held = held
|
|
entered, duration := tierLeftOf(old, *p, time.Now())
|
|
record.TierEntered, record.Updated = entered, time.Now().UTC()
|
|
if epoch != nil {
|
|
record.Epoch = uint64(*epoch)
|
|
} else {
|
|
record.Epoch = 0
|
|
}
|
|
body, merr := json.Marshal(record)
|
|
if merr != nil {
|
|
return merr
|
|
}
|
|
var rev uint64
|
|
var werr error
|
|
if found {
|
|
rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision)
|
|
} else {
|
|
rev, werr = createRaw(ctx, c, p.ID, body)
|
|
}
|
|
if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) {
|
|
if !found {
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
|
}
|
|
continue
|
|
}
|
|
if werr != nil {
|
|
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr)
|
|
}
|
|
v.apply(p.ID, body, rev)
|
|
w = written{revision: record.Revision, entered: entered, held: held, duration: duration}
|
|
break
|
|
}
|
|
done[p.ID] = w
|
|
}
|
|
|
|
// The records that gave a merge up, now it is held by the one it went to.
|
|
for id, e := range edits {
|
|
if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) {
|
|
continue
|
|
}
|
|
e := e
|
|
if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
|
return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
i.moved.mu.Lock()
|
|
for key, pm := range i.moved.pending {
|
|
if _, saved := done[pm.plan]; saved {
|
|
delete(i.moved.pending, key)
|
|
}
|
|
}
|
|
i.moved.mu.Unlock()
|
|
for _, p := range plans {
|
|
w := done[p.ID]
|
|
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
|
|
if epoch != nil {
|
|
p.Epoch = uint64(*epoch)
|
|
} else {
|
|
p.Epoch = 0
|
|
}
|
|
if w.duration != nil {
|
|
// After the write: the key of the duration is the plan and its tier, so a measurement made twice is one.
|
|
record := i.RecordDuration
|
|
if i.recorded != nil {
|
|
record = i.recorded
|
|
}
|
|
if derr := record(ctx, *w.duration); derr != nil {
|
|
fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr)
|
|
}
|
|
}
|
|
if PlanSaved != nil {
|
|
PlanSaved(i, *p)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func planNamed(plans []*Plan, id string) bool {
|
|
for _, p := range plans {
|
|
if p.ID == id {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// createRaw writes a plan's first record.
|
|
func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) {
|
|
return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body)
|
|
}
|
|
|
|
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
|
|
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
|
|
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
|
|
c, on, err := i.ChainReads(ctx, KindRollouts)
|
|
if err != nil || !on {
|
|
return 0, err
|
|
}
|
|
v, err := i.rolloutView(ctx, c)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
repaired := 0
|
|
for key, copies := range mergeHolders(v.list()) {
|
|
if len(copies) < 2 {
|
|
continue
|
|
}
|
|
w := winner(copies)
|
|
for _, h := range copies {
|
|
if h.plan == w.plan {
|
|
continue
|
|
}
|
|
key := key
|
|
if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
|
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
|
|
}); err != nil {
|
|
return repaired, err
|
|
}
|
|
repaired++
|
|
}
|
|
}
|
|
return repaired, nil
|
|
}
|