Files
mesh-controller/internal/inventory/chain_rollouts.go
T
jschoubben 275ddd76e8 Keep rollouts and the merges they answer in the rollouts bucket (issue 1092)
Once chain_moved records the kind, a plan is a key of the rollouts bucket under its id and the open batch is the same key in state assembling. A save refuses a copy another save moved since (the plan's own Revision, kept in its value, counts saves) and keeps the merges the record holds, so a copy read before a merge moved cannot lay it back. The merges a batch or walk answers are the record's held field, with the columns the table had; a merge no plan answers stays, flagged, in the record that held it last, or in the record kept for those. A merge moved to a plan is written to the receiving record first and then taken from the giving one; one found in both is read from the one it entered last and repaired at the next cut. Plans and merges are read from a view kept by one watch of the bucket. A bus that does not answer, a lease lost and a key moved are now not-now like the store being away, so a merge whose plan could not be written is held and handed over again, never acknowledged. The start moves release_plan and batched_merge into the bucket, one kind, counted against both tables.
2026-10-18 03:06:50 +02:00

940 lines
27 KiB
Go

package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan
// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk
// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside.
//
// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value,
// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no
// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the
// merge operations alone; a save keeps what the record holds.
// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the
// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by
// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one.
type rolloutView struct {
chain *chain.Chain
mu sync.RWMutex
entries map[string]viewEntry
ready chan struct{}
once sync.Once
err error
dead atomic.Bool
cancel context.CancelFunc
}
// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out.
type viewEntry struct {
raw []byte
rev uint64
sum planSummary
}
// planSummary is the part of a plan the surveys filter and sort by.
type planSummary struct {
ID string `json:"id"`
State string `json:"state"`
Created time.Time `json:"created"`
Repository string `json:"repository"`
Branch string `json:"branch"`
Merged time.Time `json:"merged"`
Release json.RawMessage `json:"release"`
Held []chain.HeldMerge `json:"held"`
}
func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling }
func (s planSummary) isRelease() bool {
r := strings.TrimSpace(string(s.Release))
return r != "" && r != "null"
}
func newRolloutView(c *chain.Chain) *rolloutView {
return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})}
}
func (v *rolloutView) start() {
ctx, cancel := context.WithCancel(context.Background())
v.cancel = cancel
go func() {
w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx)
if err != nil {
v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err))
return
}
for {
select {
case <-ctx.Done():
v.fail(ctx.Err())
return
case e, open := <-w:
if !open {
v.fail(errors.New("the rollouts bucket stopped answering"))
return
}
if e.Caught {
v.once.Do(func() { close(v.ready) })
continue
}
v.apply(e.Key, e.Value, e.Revision)
}
}
}()
}
// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read.
func (v *rolloutView) fail(err error) {
v.mu.Lock()
if v.err == nil {
v.err = err
}
v.mu.Unlock()
v.dead.Store(true)
v.once.Do(func() { close(v.ready) })
}
func (v *rolloutView) stop() {
if v.cancel != nil {
v.cancel()
}
}
// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one.
func (v *rolloutView) apply(key string, raw []byte, rev uint64) {
var sum planSummary
if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" {
sum = planSummary{ID: key}
}
v.mu.Lock()
defer v.mu.Unlock()
if old, has := v.entries[key]; has && old.rev >= rev {
return
}
v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum}
}
func (v *rolloutView) list() []viewEntry {
v.mu.RLock()
defer v.mu.RUnlock()
out := make([]viewEntry, 0, len(v.entries))
for _, e := range v.entries {
out = append(out, e)
}
return out
}
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
i.moved.mu.Lock()
v := i.moved.view
if v == nil || v.chain != c || v.dead.Load() {
if v != nil {
v.stop()
}
v = newRolloutView(c)
i.moved.view = v
v.start()
}
i.moved.mu.Unlock()
select {
case <-v.ready:
case <-ctx.Done():
return nil, ctx.Err()
}
v.mu.RLock()
defer v.mu.RUnlock()
if v.err != nil && len(v.entries) == 0 {
return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err)
}
return v, nil
}
// stopViews ends the view's watch: the inventory is closing.
func (i *Inventory) stopViews() {
i.moved.mu.Lock()
defer i.moved.mu.Unlock()
if i.moved.view != nil {
i.moved.view.stop()
i.moved.view = nil
}
}
func normalizePlan(p *Plan) {
if p.Modules == nil {
p.Modules = map[string]*PlanModule{}
}
if p.TierEntered.IsZero() {
p.TierEntered = p.Created
}
if !p.Merged.IsZero() {
p.Merged = p.Merged.UTC()
}
}
func planOf(e viewEntry) (Plan, error) {
var p Plan
if err := json.Unmarshal(e.raw, &p); err != nil {
return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err)
}
normalizePlan(&p)
return p, nil
}
func sortCreated(entries []viewEntry) {
sort.SliceStable(entries, func(a, b int) bool {
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
return entries[a].sum.Created.Before(entries[b].sum.Created)
}
return entries[a].sum.ID < entries[b].sum.ID
})
}
func sortCreatedNewest(entries []viewEntry) {
sort.SliceStable(entries, func(a, b int) bool {
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
return entries[a].sum.Created.After(entries[b].sum.Created)
}
return entries[a].sum.ID > entries[b].sum.ID
})
}
// chainPlans is the plans that pass keep, in an order, read from the view.
func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) {
v, err := i.rolloutView(ctx, c)
if err != nil {
return nil, err
}
var chosen []viewEntry
for _, e := range v.list() {
if e.sum.ID != WaitingRecord && keep(e) {
chosen = append(chosen, e)
}
}
order(chosen)
out := make([]Plan, 0, len(chosen))
for _, e := range chosen {
p, err := planOf(e)
if err != nil {
return nil, err
}
out = append(out, p)
}
return out, nil
}
func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) {
v, err := i.rolloutView(ctx, c)
if err != nil {
return Plan{}, false, err
}
var chosen []viewEntry
for _, e := range v.list() {
if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() {
chosen = append(chosen, e)
}
}
sort.SliceStable(chosen, func(a, b int) bool {
if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) {
return chosen[a].sum.Merged.After(chosen[b].sum.Merged)
}
return chosen[a].sum.Created.After(chosen[b].sum.Created)
})
if len(chosen) == 0 {
return Plan{}, false, nil
}
p, err := planOf(chosen[0])
return p, err == nil, err
}
// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view
// learns of it.
func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) {
if !chain.ValidKey(id) {
// No plan has an id the bus takes no spelling of; it is a plain id no plan has.
return Plan{}, fmt.Errorf("no plan %s", id)
}
e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id)
if err != nil {
return Plan{}, err
}
if !found {
return Plan{}, fmt.Errorf("no plan %s", id)
}
if v, err := i.rolloutView(ctx, c); err == nil {
v.apply(id, e.Value, e.Revision)
}
return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}})
}
// ---- the merges
// mergeHolder is a record's copy of a merge.
type mergeHolder struct {
plan string
m chain.HeldMerge
rev uint64
}
func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit }
// mergeHolders is every record's copy of each merge, by the merge's key.
func mergeHolders(entries []viewEntry) map[string][]mergeHolder {
out := map[string][]mergeHolder{}
for _, e := range entries {
for _, m := range e.sum.Held {
k := mergeKey(m.Repository, m.Commit)
out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev})
}
}
return out
}
// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two
// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers
// before one no plan does, then the record written last.
func winner(copies []mergeHolder) mergeHolder {
best := copies[0]
for _, c := range copies[1:] {
switch {
case c.m.Placed.After(best.m.Placed):
best = c
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting:
best = c
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev:
best = c
}
}
return best
}
func winners(entries []viewEntry) map[string]mergeHolder {
out := map[string]mergeHolder{}
for k, copies := range mergeHolders(entries) {
out[k] = winner(copies)
}
return out
}
func batchedOf(h mergeHolder) BatchedMerge {
b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit,
Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone}
if !h.m.Merged.IsZero() {
b.Merged = h.m.Merged.UTC()
}
if h.m.Waiting {
b.Plan = ""
}
return b
}
func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge {
h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(),
Event: m.Event, Alone: m.Alone, Placed: now.UTC()}
if !m.Merged.IsZero() {
h.Merged = m.Merged.UTC()
}
return h
}
func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) {
v, err := i.rolloutView(ctx, c)
if err != nil {
return nil, err
}
return winners(v.list()), nil
}
func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return BatchedMerge{}, false, err
}
h, ok := w[mergeKey(repository, commit)]
if !ok {
return BatchedMerge{}, false, nil
}
return batchedOf(h), true, nil
}
// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard.
func sortMerges(ms []BatchedMerge, newest bool) {
sort.SliceStable(ms, func(a, b int) bool {
x, y := ms[a], ms[b]
switch {
case x.Merged.IsZero() != y.Merged.IsZero():
return !x.Merged.IsZero()
case !x.Merged.Equal(y.Merged):
if newest {
return x.Merged.After(y.Merged)
}
return x.Merged.Before(y.Merged)
}
if newest {
return x.Heard.After(y.Heard)
}
return x.Heard.Before(y.Heard)
})
}
func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return nil, err
}
var out []BatchedMerge
for _, h := range w {
if h.plan == plan && !h.m.Waiting {
out = append(out, batchedOf(h))
}
}
sortMerges(out, false)
return out, nil
}
func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return nil, err
}
var out []BatchedMerge
for _, h := range w {
if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) {
out = append(out, batchedOf(h))
}
}
sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) })
return out, nil
}
func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return false, time.Time{}, err
}
var first time.Time
for _, h := range w {
if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) {
first = h.m.Heard
}
}
if first.IsZero() {
return false, time.Time{}, nil
}
return true, first.UTC(), nil
}
func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) {
w, err := i.chainWinners(ctx, c)
if err != nil {
return nil, err
}
var out []BatchedMerge
for _, h := range w {
if h.m.Waiting && h.m.Alone {
out = append(out, batchedOf(h))
}
}
sortMerges(out, true)
return out, nil
}
// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it.
type pendingMerge struct {
plan string
m chain.HeldMerge
}
// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has
// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is
// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3).
func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) {
waiting := m.Plan == ""
if waiting {
// A merge no plan answers is held, flagged, by the record kept for those.
m.Plan = WaitingRecord
if err := i.ensureWaitingRecord(ctx, c); err != nil {
return false, err
}
}
v, err := i.rolloutView(ctx, c)
if err != nil {
return false, err
}
key := mergeKey(m.Repository, m.Commit)
if _, known := winners(v.list())[key]; known {
return false, nil
}
held := heldOf(m, time.Now())
held.Waiting = waiting
found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
for _, x := range h {
if mergeKey(x.Repository, x.Commit) == key {
return h, false
}
}
return append(h, held), true
})
if err != nil {
return false, err
}
if !found {
i.moved.mu.Lock()
if i.moved.pending == nil {
i.moved.pending = map[string]pendingMerge{}
}
i.moved.pending[key] = pendingMerge{plan: m.Plan, m: held}
i.moved.mu.Unlock()
}
return true, nil
}
// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store
// kept as a row with a null plan. It is no plan: the surveys of plans leave it out.
const WaitingRecord = "waiting-merges"
// ensureWaitingRecord makes the record of the merges no plan answers, once.
func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error {
b := chain.RolloutsOf[Plan](c)
if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found {
return err
}
body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
if err != nil {
return err
}
rev, err := b.CreateRaw(ctx, WaitingRecord, body)
if errors.Is(err, chain.ErrExists) {
return nil
}
if err != nil {
return err
}
if v, verr := i.rolloutView(ctx, c); verr == nil {
v.apply(WaitingRecord, body, rev)
}
return nil
}
// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it.
var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved")
// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again
// when it moved. It answers false when the plan has no record.
func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) {
b := chain.RolloutsOf[Plan](c)
for try := 0; try < casTries; try++ {
e, found, err := b.GetRaw(ctx, id)
if err != nil || !found {
return false, err
}
var record map[string]json.RawMessage
if err := json.Unmarshal(e.Value, &record); err != nil {
return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err)
}
var held []chain.HeldMerge
if raw, has := record["held"]; has {
if err := json.Unmarshal(raw, &held); err != nil {
return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err)
}
}
next, changed := edit(held)
if !changed {
return true, nil
}
if len(next) == 0 {
delete(record, "held")
} else {
raw, err := json.Marshal(next)
if err != nil {
return true, err
}
record["held"] = raw
}
body, err := json.Marshal(record)
if err != nil {
return true, err
}
rev, err := b.UpdateRaw(ctx, id, body, e.Revision)
if errors.Is(err, chain.ErrMoved) {
continue
}
if err != nil {
return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err)
}
if v, verr := i.rolloutView(ctx, c); verr == nil {
v.apply(id, body, rev)
}
return true, nil
}
return true, fmt.Errorf("the merges of plan %s kept moving under the write", id)
}
// recordEdit is what the merges of one record gain, lose and have flagged in one act.
type recordEdit struct {
add []chain.HeldMerge
drop map[string]bool
flag map[string]chain.HeldMerge
}
func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge {
out := make([]chain.HeldMerge, 0, len(held)+len(e.add))
for _, h := range held {
k := mergeKey(h.Repository, h.Commit)
if e.drop[k] {
continue
}
if f, has := e.flag[k]; has {
h.Waiting, h.Alone = f.Waiting, f.Alone
}
out = append(out, h)
}
for _, a := range e.add {
k := mergeKey(a.Repository, a.Commit)
dup := false
for _, h := range out {
if mergeKey(h.Repository, h.Commit) == k {
dup = true
break
}
}
if !dup {
out = append(out, a)
}
}
return out
}
func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 }
// ---- the save
// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save
// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one.
func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) {
if old == nil {
return now, nil // a new plan enters its first tier now
}
if old.State == PlanAssembling || old.State == PlanQueued {
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
}
since := old.TierEntered
if since.IsZero() {
since = old.Created
}
wasOpen := old.State == PlanBuilding || old.State == PlanRolling
if !wasOpen || (old.Tier == p.Tier && p.Open()) {
return since, nil // still in the tier, or already ended
}
var modules []string
if old.Tier >= 0 && old.Tier < len(p.Tiers) {
modules = p.Tiers[old.Tier]
}
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State)
if p.Open() {
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier)
}
return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier),
Started: since, Took: now.Sub(since), Detail: detail}
}
// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by
// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is
// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a
// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092).
func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) {
epoch, err := i.actingEpoch(ctx)
if err != nil {
if len(plans) > 0 {
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
}
return err
}
v, err := i.rolloutView(ctx, c)
if err != nil {
return err
}
b := chain.RolloutsOf[Plan](c)
marked := time.Now()
undo := make([]func(), 0, len(plans))
defer func() {
if err != nil {
for _, u := range undo {
u()
}
}
}()
for _, p := range plans {
undo = append(undo, p.markWalk(marked))
}
// Every plan still at the revision it was read at, before anything is written.
cur := make(map[string]chain.Kept[Plan], len(plans))
for _, p := range plans {
k, found, err := b.Get(ctx, p.ID)
if err != nil {
return err
}
if found {
if k.Value.Revision != p.Revision {
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
cur[p.ID] = k
}
}
// What each record gains, loses and has flagged.
now := time.Now().UTC()
edits := map[string]*recordEdit{}
editOf := func(id string) *recordEdit {
if edits[id] == nil {
edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}}
}
return edits[id]
}
copies := mergeHolders(v.list())
for _, m := range moves {
key := mergeKey(m.Repository, m.Commit)
held, has := copies[key]
if !has {
// A merge carried for a plan not saved yet goes where it is moved.
i.moved.mu.Lock()
if pm, pending := i.moved.pending[key]; pending && m.Plan != "" {
pm.plan, pm.m.Alone = m.Plan, m.Alone
i.moved.pending[key] = pm
}
i.moved.mu.Unlock()
continue
}
w := winner(held)
if m.Plan == "" {
f := w.m
f.Waiting, f.Alone = true, m.Alone
editOf(w.plan).flag[key] = f
continue
}
if w.plan == m.Plan {
f := w.m
f.Waiting, f.Alone = false, m.Alone
editOf(w.plan).flag[key] = f
continue
}
moved := w.m
moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now
editOf(m.Plan).add = append(editOf(m.Plan).add, moved)
for _, h := range held {
editOf(h.plan).drop[key] = true
}
}
// The records not among the plans: receivers now, givers after.
for id, e := range edits {
if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 {
continue
}
e := e
found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
return (&recordEdit{add: e.add}).apply(h), true
})
if err != nil {
return err
}
if !found {
// Nothing is taken from the record that holds a merge to give it to a plan that has none.
return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add))
}
}
// The plans, those that gain a merge first.
order := append([]*Plan(nil), plans...)
sort.SliceStable(order, func(a, b int) bool {
x, y := edits[order[a].ID], edits[order[b].ID]
return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0)
})
type written struct {
revision int64
entered time.Time
held []chain.HeldMerge
duration *Duration
}
done := map[string]written{}
for _, p := range order {
var w written
for try := 0; ; try++ {
if try == casTries {
return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID)
}
k, found := cur[p.ID]
if try > 0 {
fresh, still, gerr := b.Get(ctx, p.ID)
if gerr != nil {
return gerr
}
if still && fresh.Value.Revision != p.Revision {
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
k, found = fresh, still
cur[p.ID] = k
}
record := *p
var old *Plan
var held []chain.HeldMerge
if found {
o := k.Value
old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...)
record.Revision = k.Value.Revision + 1
} else {
record.Revision = 1
}
if e := edits[p.ID]; e != nil {
held = e.apply(held)
}
i.moved.mu.Lock()
for key, pm := range i.moved.pending {
if pm.plan == p.ID {
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
_ = key
}
}
i.moved.mu.Unlock()
record.Held = held
entered, duration := tierLeftOf(old, *p, time.Now())
record.TierEntered, record.Updated = entered, time.Now().UTC()
if epoch != nil {
record.Epoch = uint64(*epoch)
} else {
record.Epoch = 0
}
body, merr := json.Marshal(record)
if merr != nil {
return merr
}
var rev uint64
var werr error
if found {
rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision)
} else {
rev, werr = createRaw(ctx, c, p.ID, body)
}
if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) {
if !found {
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
}
continue
}
if werr != nil {
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr)
}
v.apply(p.ID, body, rev)
w = written{revision: record.Revision, entered: entered, held: held, duration: duration}
break
}
done[p.ID] = w
}
// The records that gave a merge up, now it is held by the one it went to.
for id, e := range edits {
if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) {
continue
}
e := e
if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true
}); err != nil {
return err
}
}
i.moved.mu.Lock()
for key, pm := range i.moved.pending {
if _, saved := done[pm.plan]; saved {
delete(i.moved.pending, key)
}
}
i.moved.mu.Unlock()
for _, p := range plans {
w := done[p.ID]
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
if epoch != nil {
p.Epoch = uint64(*epoch)
} else {
p.Epoch = 0
}
if w.duration != nil {
// After the write: the key of the duration is the plan and its tier, so a measurement made twice is one.
record := i.RecordDuration
if i.recorded != nil {
record = i.recorded
}
if derr := record(ctx, *w.duration); derr != nil {
fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr)
}
}
if PlanSaved != nil {
PlanSaved(i, *p)
}
}
return nil
}
func planNamed(plans []*Plan, id string) bool {
for _, p := range plans {
if p.ID == id {
return true
}
}
return false
}
// createRaw writes a plan's first record.
func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) {
return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body)
}
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
c, on, err := i.ChainReads(ctx, KindRollouts)
if err != nil || !on {
return 0, err
}
v, err := i.rolloutView(ctx, c)
if err != nil {
return 0, err
}
repaired := 0
for key, copies := range mergeHolders(v.list()) {
if len(copies) < 2 {
continue
}
w := winner(copies)
for _, h := range copies {
if h.plan == w.plan {
continue
}
key := key
if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
}); err != nil {
return repaired, err
}
repaired++
}
}
return repaired, nil
}