1168 lines
46 KiB
Go
1168 lines
46 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
|
|
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
|
)
|
|
|
|
// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is
|
|
// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another
|
|
// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it.
|
|
const (
|
|
// PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed
|
|
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
|
|
// 0595 §11), history 1, reached by the indexer alone.
|
|
PendingState = "pending"
|
|
PendingBucket = delivery.IndexerModule + "_" + PendingState
|
|
// PendingTool reads it (ADR 0633 §5); TakeInPendingTool and DropPendingTool steer it.
|
|
PendingTool = "indexer_pending"
|
|
TakeInPendingTool = "indexer_take_in_pending"
|
|
DropPendingTool = "indexer_drop_pending"
|
|
)
|
|
|
|
// errClaimLost says another take-in claimed a pending merge first: this one took nothing in.
|
|
var errClaimLost = errors.New("another take-in claimed the merges first, so this one took nothing in")
|
|
|
|
// NoBuild opens what a revision waits for when it gives the indexer no fingerprint: the reason follows. No indexer
|
|
// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending).
|
|
const NoBuild = "no build: "
|
|
|
|
// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1):
|
|
// the message is left unacknowledged, so the bus delivers it again, and nothing is written.
|
|
var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known")
|
|
|
|
// PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many
|
|
// modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out
|
|
// from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it.
|
|
type PendingRef struct {
|
|
Repository string `json:"repository"`
|
|
Number int `json:"number,omitempty"`
|
|
Merge string `json:"merge"`
|
|
// Branch is the pull request's branch, which names its candidate: a merge covered by a newer one is taken out of it.
|
|
Branch string `json:"branch,omitempty"`
|
|
Members []delivery.Head `json:"members,omitempty"`
|
|
// Holds are the modules the merge's manifests name, which indexer_pending says it holds back.
|
|
Holds []string `json:"holds,omitempty"`
|
|
}
|
|
|
|
// Pending is one value of the pending state: a reference to the merged message (Seq, its sequence on the forge bridge's
|
|
// stream, and Ref), or, in a value an earlier indexer wrote, the message whole as the forge bridge encoded it; the group it
|
|
// was merged in (Ended once the group's end was heard), since when it is held, the indexer build its revision waits
|
|
// for (absent while it waits for nothing yet, or behind a group being gathered), the build of the indexer that last
|
|
// wrote it, the wait it was last said under on the take-in stream (Said, SaidAt), and Taking, set by a take-in that
|
|
// claimed it: a second indexer that finds it claimed takes nothing in (ADR 0633 §1: never two take-ins of one merge).
|
|
type Pending struct {
|
|
V int `json:"v"`
|
|
Seq uint64 `json:"seq,omitempty"`
|
|
Ref *PendingRef `json:"ref,omitempty"`
|
|
Forge json.RawMessage `json:"forge,omitempty"`
|
|
Lost string `json:"lost,omitempty"`
|
|
Group string `json:"group,omitempty"`
|
|
Ended bool `json:"ended,omitempty"`
|
|
Since time.Time `json:"since"`
|
|
WaitsFor string `json:"waits-for,omitempty"`
|
|
LastWriter string `json:"last-writer"`
|
|
Said string `json:"said,omitempty"`
|
|
SaidKind string `json:"said-kind,omitempty"`
|
|
SaidAt time.Time `json:"said-at,omitzero"`
|
|
Taking string `json:"taking,omitempty"`
|
|
}
|
|
|
|
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
|
|
// value that keeps a reference, Forge holds only the fields of the reference (Light) until Indexer.load reads the message
|
|
// from the stream; Holds are the modules the merge's manifests name.
|
|
type PendingEntry struct {
|
|
Key string
|
|
Revision uint64
|
|
Value Pending
|
|
Forge forgemsg.Forge
|
|
Light bool
|
|
Holds []string
|
|
// Covered says a newer merge of the same repository is in the same revision: its message is not read or applied, since
|
|
// applying the newest writes every module whose manifest differs from what is held (modules.go).
|
|
Covered bool
|
|
}
|
|
|
|
// DecodePending reads one pending value and its merged message.
|
|
func DecodePending(data []byte) (Pending, forgemsg.Forge, error) {
|
|
var p Pending
|
|
if err := json.Unmarshal(data, &p); err != nil {
|
|
return p, forgemsg.Forge{}, fmt.Errorf("not a pending value: %w", err)
|
|
}
|
|
if p.V != 1 {
|
|
return p, forgemsg.Forge{}, fmt.Errorf("a pending value of version %d, not 1", p.V)
|
|
}
|
|
if len(p.Forge) == 0 && p.Ref != nil {
|
|
r := p.Ref
|
|
return p, forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: r.Repository, Number: r.Number,
|
|
Merge: r.Merge, Branch: r.Branch, Group: p.Group, Members: r.Members}, nil
|
|
}
|
|
f, err := forgemsg.DecodeForge(p.Forge)
|
|
if err != nil {
|
|
return p, f, fmt.Errorf("its merged message: %w", err)
|
|
}
|
|
if f.Kind != forgemsg.ForgeMerged {
|
|
return p, f, fmt.Errorf("a pending value holds a %s message, not a merged one", f.Kind)
|
|
}
|
|
return p, f, nil
|
|
}
|
|
|
|
func encodePending(p Pending) ([]byte, error) {
|
|
p.V = 1
|
|
return json.Marshal(p)
|
|
}
|
|
|
|
// ReadPending is every value the pending state holds, in key order (the merge order); a key that does not read is
|
|
// said through logf and left.
|
|
func ReadPending(ctx context.Context, st Store, logf func(string, ...any)) ([]PendingEntry, error) {
|
|
keys, err := st.Keys(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("listing %s: %w", PendingState, err)
|
|
}
|
|
slices.Sort(keys)
|
|
var out []PendingEntry
|
|
for _, key := range keys {
|
|
value, rev, found, err := st.Get(ctx, key)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s.%s: %w", PendingState, key, err)
|
|
}
|
|
if !found {
|
|
continue
|
|
}
|
|
p, f, err := DecodePending(value)
|
|
if err != nil {
|
|
if logf != nil {
|
|
logf("%s.%s is left: %v", PendingState, key, err)
|
|
}
|
|
continue
|
|
}
|
|
e := PendingEntry{Key: key, Revision: rev, Value: p, Forge: f, Light: len(p.Forge) == 0}
|
|
if p.Ref != nil {
|
|
e.Holds = p.Ref.Holds
|
|
} else {
|
|
e.Holds = holdsOf(f)
|
|
}
|
|
out = append(out, e)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// holdsOf are the modules a merged message's manifests name, sorted.
|
|
func holdsOf(f forgemsg.Forge) []string {
|
|
var out []string
|
|
for _, raw := range f.Manifests {
|
|
if name := manifestModule(raw); name != "" && !slices.Contains(out, name) {
|
|
out = append(out, name)
|
|
}
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// refOf is what a pending value keeps of the merged message f (PendingRef).
|
|
func refOf(f forgemsg.Forge) *PendingRef {
|
|
return &PendingRef{Repository: f.Repository, Number: f.Number, Merge: f.Merge, Branch: f.Branch, Members: f.Members, Holds: holdsOf(f)}
|
|
}
|
|
|
|
// heldMessage is the message Arrive was handed for a sequence, kept in memory until its merge is taken in or dropped.
|
|
func (ix *Indexer) heldMessage(seq uint64) (forgemsg.Forge, bool) {
|
|
ix.heldMu.Lock()
|
|
defer ix.heldMu.Unlock()
|
|
f, ok := ix.held[seq]
|
|
return f, ok
|
|
}
|
|
|
|
func (ix *Indexer) hold(seq uint64, f forgemsg.Forge) {
|
|
ix.heldMu.Lock()
|
|
defer ix.heldMu.Unlock()
|
|
if ix.held == nil {
|
|
ix.held = map[uint64]forgemsg.Forge{}
|
|
}
|
|
ix.held[seq] = f
|
|
}
|
|
|
|
func (ix *Indexer) forget(entries ...PendingEntry) {
|
|
ix.heldMu.Lock()
|
|
defer ix.heldMu.Unlock()
|
|
for _, e := range entries {
|
|
delete(ix.held, e.Value.Seq)
|
|
}
|
|
}
|
|
|
|
// load gives the entry the merged message whole: the copy an earlier value holds, the message Arrive was handed (kept in
|
|
// memory), else the message read from the forge bridge's stream by the value's sequence. A stream that no longer holds it
|
|
// answers ErrForgeGone, a bus that refuses the read its error; a message that is not the merge the value refers to (a
|
|
// stream rebuilt under the same sequence) is refused.
|
|
func (ix *Indexer) load(ctx context.Context, e PendingEntry) (PendingEntry, error) {
|
|
if !e.Light {
|
|
return e, nil
|
|
}
|
|
if f, ok := ix.heldMessage(e.Value.Seq); ok {
|
|
e.Forge, e.Light = f, false
|
|
return e, nil
|
|
}
|
|
if ix.Message == nil {
|
|
return e, fmt.Errorf("the forge bridge is not asked by this indexer, so %s (%s#%d) cannot be read", e.Key, e.Forge.Repository, e.Forge.Number)
|
|
}
|
|
data, err := ix.Message(ctx, e.Forge.Repository, e.Forge.Merge)
|
|
if err != nil {
|
|
return e, fmt.Errorf("the forge bridge could not answer for the merge of %s#%d at %s: %w", e.Forge.Repository, e.Forge.Number, e.Forge.Merge, err)
|
|
}
|
|
f, err := forgemsg.DecodeForge(data)
|
|
if err != nil {
|
|
return e, fmt.Errorf("the forge bridge's answer for %s is not a forge message: %w", e.Key, err)
|
|
}
|
|
if f.Kind != forgemsg.ForgeMerged || f.Repository != e.Forge.Repository || f.Merge != e.Forge.Merge {
|
|
return e, fmt.Errorf("the forge bridge's answer is not the merge %s#%d at %s that %s refers to", e.Forge.Repository, e.Forge.Number, e.Forge.Merge, e.Key)
|
|
}
|
|
e.Forge, e.Light = f, false
|
|
return e, nil
|
|
}
|
|
|
|
// loadNewest prepares the ready merges of one revision, in merge order. Of the merges of one repository only the newest is
|
|
// loaded: applying it writes every module whose manifest differs from what is held, so it covers the older ones, which are
|
|
// marked Covered and never read. A newest merge the forge bridge cannot give is answered in lost with its reason, by key, and its revision is held (holdLost).
|
|
func (ix *Indexer) loadNewest(ctx context.Context, entries []PendingEntry) (out, lost []PendingEntry, reasons map[string]string) {
|
|
out = slices.Clone(entries)
|
|
reasons = map[string]string{}
|
|
newest := map[string]bool{}
|
|
for i := len(out) - 1; i >= 0; i-- {
|
|
repo := strings.ToLower(out[i].Forge.Repository)
|
|
if newest[repo] {
|
|
out[i].Covered = true
|
|
continue
|
|
}
|
|
newest[repo] = true
|
|
full, err := ix.load(ctx, out[i])
|
|
if err != nil {
|
|
lost = append(lost, out[i])
|
|
reasons[out[i].Key] = err.Error()
|
|
continue
|
|
}
|
|
out[i] = full
|
|
}
|
|
slices.Reverse(lost)
|
|
return out, lost, reasons
|
|
}
|
|
|
|
// Waits is the pending state read as what waits (ADR 0633 §3, §4): Ready are the merges no group still being gathered
|
|
// holds back, in merge order, which are one revision together; Groups are the groups still being gathered, each with
|
|
// its members held so far; Behind are the merges in a gathered group's member repositories that wait behind it, by the
|
|
// group's take-in key.
|
|
type Waits struct {
|
|
Ready []PendingEntry
|
|
Groups map[string][]PendingEntry
|
|
Behind map[string][]PendingEntry
|
|
}
|
|
|
|
// WaitsOf works out what waits from the entries, in key order. A member of a group whose end was not heard is
|
|
// gathered; from the first such member on, a later merge in one of the group's members' repositories waits behind it,
|
|
// because its tree holds that member's commit. Every other merge is ready.
|
|
func WaitsOf(entries []PendingEntry) Waits {
|
|
w := Waits{Groups: map[string][]PendingEntry{}, Behind: map[string][]PendingEntry{}}
|
|
blocked := map[string]string{} // repository, lower case, to the group key it waits behind
|
|
for _, e := range entries {
|
|
if e.Value.Group != "" && !e.Value.Ended {
|
|
w.Groups[e.Value.Group] = append(w.Groups[e.Value.Group], e)
|
|
for _, m := range e.Forge.Members {
|
|
if _, ok := blocked[strings.ToLower(m.Repository)]; !ok {
|
|
blocked[strings.ToLower(m.Repository)] = takein.GroupTakeInKey(e.Value.Group)
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if g, ok := blocked[strings.ToLower(e.Forge.Repository)]; ok {
|
|
w.Behind[g] = append(w.Behind[g], e)
|
|
continue
|
|
}
|
|
w.Ready = append(w.Ready, e)
|
|
}
|
|
return w
|
|
}
|
|
|
|
// mergesOf are the merges of entries, oldest first.
|
|
func mergesOf(entries []PendingEntry) []takein.Merge {
|
|
out := make([]takein.Merge, len(entries))
|
|
for i, e := range entries {
|
|
out[i] = takein.Merge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func sinceOf(entries []PendingEntry) time.Time {
|
|
var at time.Time
|
|
for _, e := range entries {
|
|
if at.IsZero() || e.Value.Since.Before(at) {
|
|
at = e.Value.Since
|
|
}
|
|
}
|
|
return at
|
|
}
|
|
|
|
// groupWait is the take-in body of a group being gathered.
|
|
func groupWait(kind, group string, held []PendingEntry) takein.TakeIn {
|
|
t := takein.TakeIn{Kind: kind, Key: takein.GroupTakeInKey(group), What: takein.TakeInGroup, Group: group,
|
|
Merges: mergesOf(held), Since: sinceOf(held)}
|
|
if len(held) > 0 {
|
|
t.Members = held[len(held)-1].Forge.Members
|
|
}
|
|
return t
|
|
}
|
|
|
|
// revisionWait is the take-in body of a revision: its key is its oldest merge's.
|
|
func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.TakeIn {
|
|
return takein.TakeIn{Kind: kind, Key: ready[0].Key, What: takein.TakeInRevision, Merges: mergesOf(ready),
|
|
WaitsFor: waitsFor, Since: sinceOf(ready)}
|
|
}
|
|
|
|
// pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears
|
|
// (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read
|
|
// that fails otherwise counts as on the bus: the step that follows answers the error.
|
|
func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
|
|
_, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0))
|
|
return !errors.Is(err, process.ErrNotYet)
|
|
}
|
|
|
|
// ownBuildNow is the build this program runs: the one learnt, else asked of the node-engine now.
|
|
func (ix *Indexer) ownBuildNow(ctx context.Context) string {
|
|
ix.ownMu.Lock()
|
|
fp := ix.ownFP
|
|
ix.ownMu.Unlock()
|
|
if fp != "" || ix.OwnBuild == nil {
|
|
return fp
|
|
}
|
|
ix.learnOwnBuild(ctx)
|
|
ix.ownMu.Lock()
|
|
defer ix.ownMu.Unlock()
|
|
return ix.ownFP
|
|
}
|
|
|
|
// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of
|
|
// each wait from the state. A publish that fails is said in the log: the next evaluation says it again.
|
|
func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool {
|
|
if ix.SayTakeIn == nil {
|
|
return true
|
|
}
|
|
if err := ix.SayTakeIn(ctx, t); err != nil {
|
|
ix.logf("take-in: %s.%s is not said: %v", t.Kind, t.Key, err)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// markSaid writes on each entry the wait it was said under, by compare-and-set; a lost one is left (its next
|
|
// evaluation says it again).
|
|
func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *takein.TakeIn, change func(*Pending)) {
|
|
for _, e := range entries {
|
|
p := e.Value
|
|
if t != nil {
|
|
p.Said, p.SaidKind, p.SaidAt = t.Key, t.Kind, ix.now()
|
|
}
|
|
if change != nil {
|
|
change(&p)
|
|
}
|
|
data, err := encodePending(p)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil {
|
|
ix.logf("take-in: %s.%s is not written again: %v", PendingState, e.Key, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Arrive takes one merged message of the forge bridge's stream in under ADR 0633: seq is its stream sequence. It is
|
|
// written to pending first (a redelivered message finds its key there and writes no second), and then what pending
|
|
// holds is evaluated (Evaluate): taken in whole when its revision names the build this program runs, or held for the
|
|
// indexer it names. It answers what was taken in, and a line saying what waits. A message is never written while the
|
|
// build this program runs is not known (ErrOwnBuildUnknown).
|
|
func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Applied, string, error) {
|
|
if ix.Pending == nil || ix.Modules == nil || !ix.pendingOnBus(ctx) {
|
|
// The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before,
|
|
// whether or not the build this program runs is known.
|
|
a, err := ix.ApplyMessage(ctx, f)
|
|
return a, "", err
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
if own == "" {
|
|
return Applied{}, "", ErrOwnBuildUnknown
|
|
}
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
key := takein.PendingKey(seq)
|
|
ix.hold(seq, f)
|
|
data, err := encodePending(Pending{Seq: seq, Ref: refOf(f), Group: f.Group, Since: ix.now(), LastWriter: own})
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
switch err := ix.Pending.Create(ctx, key, data); {
|
|
case errors.Is(err, process.ErrNotYet):
|
|
// The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before.
|
|
ix.logf("take-in: the state %s is not on the bus yet, so %s#%d is taken in at once, as before ADR 0633", PendingBucket, f.Repository, f.Number)
|
|
a, err := ix.ApplyMessage(ctx, f)
|
|
return a, "", err
|
|
case err != nil && !errors.Is(err, ErrConflict):
|
|
return Applied{}, "", fmt.Errorf("%s.%s: %w", PendingState, key, err)
|
|
}
|
|
if f.Group != "" {
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
if held := WaitsOf(entries).Groups[f.Group]; len(held) > 0 {
|
|
t := groupWait(takein.TakeInPending, f.Group, held)
|
|
if ix.sayTakeIn(ctx, t) {
|
|
ix.markSaid(ctx, held, &t, nil)
|
|
}
|
|
}
|
|
}
|
|
return ix.evaluate(ctx, own, false)
|
|
}
|
|
|
|
// EndGroup takes a group's end in (ADR 0633 §4): once every member it names is held, the group's members are marked
|
|
// ended, the group's wait is said taken (take-in.taken under its group key), and pending is evaluated, the group now
|
|
// one revision. A member named and neither held nor taken in yet answers an error, so the end is delivered again once
|
|
// the member arrived. An end of a group taken in already (every member's merge at or behind its repository's taken
|
|
// merge) changes nothing and is said: it is never delivered again for ever.
|
|
func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Applied, string, error) {
|
|
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
|
return Applied{}, "no pending state: the group's merges were taken in one by one", nil
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
if own == "" {
|
|
return Applied{}, "", ErrOwnBuildUnknown
|
|
}
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if errors.Is(err, process.ErrNotYet) {
|
|
return Applied{}, "the pending state is not on the bus yet: the group's merges were taken in one by one", nil
|
|
}
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
var held []PendingEntry
|
|
for _, e := range entries {
|
|
if e.Value.Group == end.Group {
|
|
held = append(held, e)
|
|
}
|
|
}
|
|
isHeld := func(m forgemsg.GroupMerge) bool {
|
|
return slices.ContainsFunc(held, func(e PendingEntry) bool {
|
|
return delivery.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number
|
|
})
|
|
}
|
|
var missing []string
|
|
for _, m := range end.Members {
|
|
if isHeld(m) {
|
|
continue
|
|
}
|
|
taken, err := ix.takenAlready(ctx, m)
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
if !taken {
|
|
missing = append(missing, fmt.Sprintf("%s#%d", m.Repository, m.Number))
|
|
}
|
|
}
|
|
if len(missing) > 0 {
|
|
return Applied{}, "", fmt.Errorf("the end of the group %q names %s, not held yet: it waits for their merged messages", end.Group, strings.Join(missing, ", "))
|
|
}
|
|
if len(held) == 0 {
|
|
ix.logf("take-in: the end of the group %q comes after its take-in: every member is taken in already, nothing changes", end.Group)
|
|
return Applied{}, "the group is taken in already", nil
|
|
}
|
|
if !slices.ContainsFunc(held, func(e PendingEntry) bool { return !e.Value.Ended }) {
|
|
return ix.evaluate(ctx, own, false)
|
|
}
|
|
// The group's wait ends first (take-in.taken under its group key); a revision it then is, pending for another
|
|
// indexer, is said pending under its oldest merge's key by the evaluation that follows.
|
|
t := groupWait(takein.TakeInTaken, end.Group, held)
|
|
said := &t
|
|
if !ix.sayTakeIn(ctx, t) {
|
|
said = nil // said again by the take-in or the hold that follows (saidWaits)
|
|
}
|
|
ix.markSaid(ctx, held, said, func(p *Pending) { p.Ended = true })
|
|
ix.logf("take-in: the group %q ended with %d members held: one revision now", end.Group, len(held))
|
|
return ix.evaluate(ctx, own, false)
|
|
}
|
|
|
|
// takenAlready says whether a group member's merge is at or behind its repository's taken merge.
|
|
func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (bool, error) {
|
|
if ix.Taken == nil {
|
|
return false, nil
|
|
}
|
|
key, err := delivery.RepositoryKey(m.Repository)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
value, _, found, err := ix.Taken.Get(ctx, key)
|
|
if errors.Is(err, process.ErrNotYet) || (err == nil && !found) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, fmt.Errorf("taken.%s: %w", key, err)
|
|
}
|
|
cur, err := delivery.DecodeTaken(value)
|
|
if err != nil || cur.Merge == "" {
|
|
return false, nil
|
|
}
|
|
if cur.Merge == m.Merge {
|
|
return true, nil
|
|
}
|
|
return ix.older(ctx, m.Repository, m.Merge, cur.Merge)
|
|
}
|
|
|
|
// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts:
|
|
// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a
|
|
// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an
|
|
// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for
|
|
// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the
|
|
// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer.
|
|
func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
|
|
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
|
// Nothing can wait in a state not on the bus: the start asks no build and does not wait for one.
|
|
return Applied{}, "the pending state is not on the bus yet: nothing waits", nil
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
if own == "" {
|
|
ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again")
|
|
return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil
|
|
}
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
a, words, err := ix.evaluate(ctx, own, true)
|
|
if errors.Is(err, process.ErrNotYet) {
|
|
return a, "the pending state is not on the bus yet: nothing waits", nil
|
|
}
|
|
return a, words, err
|
|
}
|
|
|
|
// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report
|
|
// (ADR 0633 §1, §3): a revision that now names the running build is taken in, and one that names another build is held
|
|
// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known.
|
|
func (ix *Indexer) PendingAgain(ctx context.Context) error {
|
|
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
|
return nil
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
if own == "" {
|
|
return nil
|
|
}
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
_, words, err := ix.evaluate(ctx, own, false)
|
|
if errors.Is(err, process.ErrNotYet) {
|
|
return nil
|
|
}
|
|
if err == nil && words != "" {
|
|
ix.logf("take-in: evaluated again after a toolchain report: %s", words)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// evaluate works out what pending's ready merges name, together (ADR 0633 §3), and takes them in or holds them.
|
|
func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applied, string, error) {
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
w := WaitsOf(entries)
|
|
if len(w.Ready) == 0 {
|
|
return Applied{}, ix.waitingLine(w), nil
|
|
}
|
|
if !start {
|
|
for _, e := range w.Ready {
|
|
if e.Value.Taking != "" {
|
|
ix.logf("take-in: %s.%s is claimed by the take-in %s, so this indexer takes nothing in", PendingState, e.Key, e.Value.Taking)
|
|
return Applied{}, "claimed by another take-in", nil
|
|
}
|
|
}
|
|
}
|
|
// Of each repository's merges only the newest is read (from memory, else asked of the forge bridge: a pending value
|
|
// keeps a reference); the older ones are covered by it. A newest merge that cannot be read is said and
|
|
// holds its revision: nothing of it is taken in silently.
|
|
ready, lost, reasons := ix.loadNewest(ctx, w.Ready)
|
|
if len(lost) > 0 {
|
|
return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons)
|
|
}
|
|
names, dry, err := ix.computeOver(ctx, ready)
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
if names == own {
|
|
a, err := ix.takeInWhole(ctx, ready, own, "")
|
|
if errors.Is(err, errClaimLost) {
|
|
return a, "claimed by another take-in", nil
|
|
}
|
|
return a, ix.waitingLine(w), err
|
|
}
|
|
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
|
|
}
|
|
|
|
// holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds
|
|
// it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry
|
|
// (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or
|
|
// indexer_drop_pending drops it. Nothing of the revision is taken in meanwhile.
|
|
func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, reasons map[string]string) error {
|
|
t := revisionWait(takein.TakeInPending, ready, "")
|
|
for _, g := range lost {
|
|
t.Lost = append(t.Lost, g.Key)
|
|
}
|
|
said := true
|
|
for _, e := range ready {
|
|
isLost := slices.Contains(t.Lost, e.Key)
|
|
if e.Value.Said != t.Key || e.Value.SaidKind != takein.TakeInPending || (isLost && e.Value.Lost == "") {
|
|
said = false
|
|
}
|
|
}
|
|
if said {
|
|
return nil
|
|
}
|
|
for _, g := range lost {
|
|
ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; a newer merge of %s covers it, indexer_drop_pending drops it",
|
|
g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], g.Forge.Repository)
|
|
}
|
|
if !ix.sayTakeIn(ctx, t) {
|
|
return nil
|
|
}
|
|
var kept []PendingEntry
|
|
for _, e := range ready {
|
|
if !slices.Contains(t.Lost, e.Key) {
|
|
kept = append(kept, e)
|
|
}
|
|
}
|
|
for _, g := range lost {
|
|
reason := reasons[g.Key]
|
|
ix.markSaid(ctx, []PendingEntry{g}, &t, func(p *Pending) { p.Lost = reason })
|
|
}
|
|
ix.markSaid(ctx, kept, &t, nil)
|
|
return nil
|
|
}
|
|
|
|
// waitingLine says what waits after an evaluation, for the heard state.
|
|
func (ix *Indexer) waitingLine(w Waits) string {
|
|
var parts []string
|
|
for _, g := range slices.Sorted(maps.Keys(w.Groups)) {
|
|
parts = append(parts, fmt.Sprintf("the group %q gathered (%d held)", g, len(w.Groups[g])))
|
|
}
|
|
for _, g := range slices.Sorted(maps.Keys(w.Behind)) {
|
|
parts = append(parts, fmt.Sprintf("%d merges behind %s", len(w.Behind[g]), g))
|
|
}
|
|
return strings.Join(parts, "; ")
|
|
}
|
|
|
|
// dryRun is an indexer computing a take-in without writing it, and the overlays of what it would have written.
|
|
type dryRun struct {
|
|
ix *Indexer
|
|
modules, graph, verdicts *overlayStore
|
|
candidates, taken, predefined, configs *overlayStore
|
|
oldSections *overlayStore
|
|
}
|
|
|
|
// dry is an indexer over overlays of this one's states (overlay.go): what a take-in would write stays in memory and
|
|
// nothing reaches the bus but the manifests and the packages the revision publishes, which clause 1 writes all the
|
|
// same. It says no event, kicks nothing in the background, and knows what this one knows of the source, the toolchains
|
|
// and the previews. Every field of Indexer is named here (TestDryNamesEveryField holds it so).
|
|
func (ix *Indexer) dry() *dryRun {
|
|
over := func(st Store) (Store, *overlayStore) {
|
|
if st == nil {
|
|
return nil, nil
|
|
}
|
|
o := newOverlay(st)
|
|
return o, o
|
|
}
|
|
d := &dryRun{}
|
|
x := &Indexer{Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
|
|
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, Read: ix.Read, Contains: ix.Contains,
|
|
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil}
|
|
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
|
|
x.Store, d.candidates = over(ix.Store)
|
|
x.Modules, d.modules = over(ix.Modules)
|
|
x.Graph, d.graph = over(ix.Graph)
|
|
x.Verdicts, d.verdicts = over(ix.Verdicts)
|
|
x.Taken, d.taken = over(ix.Taken)
|
|
x.Predefined, d.predefined = over(ix.Predefined)
|
|
x.ConfigCandidates, d.configs = over(ix.ConfigCandidates)
|
|
x.OldSections, d.oldSections = over(ix.OldSections)
|
|
ix.mu.Lock()
|
|
x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)}
|
|
x.previews = maps.Clone(ix.previews)
|
|
x.toolchains = map[string]map[string]delivery.Toolchain{}
|
|
for l, by := range ix.toolchains {
|
|
x.toolchains[l] = maps.Clone(by)
|
|
}
|
|
x.moving = maps.Clone(ix.moving)
|
|
x.untaken = maps.Clone(ix.untaken)
|
|
ix.mu.Unlock()
|
|
ix.ownMu.Lock()
|
|
x.ownFP = ix.ownFP
|
|
ix.ownMu.Unlock()
|
|
d.ix = x
|
|
return d
|
|
}
|
|
|
|
// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees
|
|
// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "",
|
|
// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read
|
|
// is left out, as its take-in will drop it.
|
|
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) {
|
|
d := ix.dry()
|
|
for _, e := range ready {
|
|
if e.Covered {
|
|
continue
|
|
}
|
|
if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil {
|
|
if errors.Is(err, ErrUnreadable) {
|
|
continue
|
|
}
|
|
return "", d, fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
|
|
}
|
|
}
|
|
key, _ := delivery.ModulesKey(delivery.IndexerModule)
|
|
value, _, found, err := d.ix.Modules.Get(ctx, key)
|
|
if err != nil {
|
|
return "", d, err
|
|
}
|
|
if !found {
|
|
return NoBuild + "the indexer holds no record of itself", d, nil
|
|
}
|
|
m, err := delivery.DecodeModule(value)
|
|
if err != nil {
|
|
return "", d, nil
|
|
}
|
|
if m.FP2 == "" {
|
|
return NoBuild + m.None2, d, nil
|
|
}
|
|
return m.FP2, d, nil
|
|
}
|
|
|
|
// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another
|
|
// take-in wrote the key, or the store failed) releases every claim this take-in made and answers false: nothing is
|
|
// taken in, and no claim of this one is left behind.
|
|
func (ix *Indexer) claim(ctx context.Context, entries []PendingEntry, own, by string) ([]PendingEntry, bool) {
|
|
id := fmt.Sprintf("%s at %s", own, ix.now().Format(time.RFC3339Nano))
|
|
if by != "" {
|
|
id = by + ": " + id
|
|
}
|
|
claimed := make([]PendingEntry, 0, len(entries))
|
|
for _, e := range entries {
|
|
p := e.Value
|
|
p.Taking, p.LastWriter = id, own
|
|
data, err := encodePending(p)
|
|
if err == nil {
|
|
err = ix.Pending.Update(ctx, e.Key, data, e.Revision)
|
|
}
|
|
if err != nil {
|
|
ix.logf("take-in: the claim of %s.%s is lost (%v): another take-in took it, so this one takes nothing in", PendingState, e.Key, err)
|
|
ix.release(ctx, claimed)
|
|
return nil, false
|
|
}
|
|
e.Value = p
|
|
_, rev, _, err := ix.Pending.Get(ctx, e.Key)
|
|
if err != nil {
|
|
ix.logf("take-in: the claim of %s.%s cannot be read back (%v), so this take-in takes nothing in", PendingState, e.Key, err)
|
|
ix.release(ctx, append(claimed, e))
|
|
return nil, false
|
|
}
|
|
e.Revision = rev
|
|
claimed = append(claimed, e)
|
|
}
|
|
return claimed, true
|
|
}
|
|
|
|
// release clears this take-in's claim on each entry, so a later merge, a restart or a hand take-in takes it in. It
|
|
// reads the key again and clears it by compare-and-set only while the claim is still this take-in's: a key another
|
|
// take-in wrote since, or removed, is left to it. A release that fails is said; the start of the next indexer voids
|
|
// the claim (TakeInAtStart).
|
|
func (ix *Indexer) release(ctx context.Context, claimed []PendingEntry) {
|
|
for _, e := range claimed {
|
|
value, rev, found, err := ix.Pending.Get(ctx, e.Key)
|
|
if err == nil && !found {
|
|
continue
|
|
}
|
|
var cur Pending
|
|
if err == nil {
|
|
err = json.Unmarshal(value, &cur)
|
|
}
|
|
if err == nil && cur.Taking != e.Value.Taking {
|
|
continue
|
|
}
|
|
if err == nil {
|
|
cur.Taking = ""
|
|
var data []byte
|
|
if data, err = encodePending(cur); err == nil {
|
|
err = ix.Pending.Update(ctx, e.Key, data, rev)
|
|
}
|
|
}
|
|
if err != nil {
|
|
ix.logf("take-in: the claim of %s.%s is not released (%v): the next indexer's start voids it", PendingState, e.Key, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// takeInWhole takes the entries in with this indexer, as merges are taken in (ApplyMessage), oldest first, then
|
|
// removes them from pending and says each wait they were said under taken. by names a hand act ("" for none).
|
|
func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own, by string) (Applied, error) {
|
|
return ix.takeInWholeBy(ctx, entries, own, by, "", "")
|
|
}
|
|
|
|
// takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events.
|
|
func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) {
|
|
claimed, ok := ix.claim(ctx, entries, own, by)
|
|
if !ok {
|
|
return Applied{}, errClaimLost
|
|
}
|
|
var applied Applied
|
|
newest := map[string]PendingEntry{}
|
|
for _, e := range claimed {
|
|
if !e.Covered {
|
|
newest[strings.ToLower(e.Forge.Repository)] = e
|
|
}
|
|
}
|
|
for _, e := range claimed {
|
|
if e.Covered {
|
|
by := newest[strings.ToLower(e.Forge.Repository)]
|
|
ix.logf("take-in: %s (%s#%d) is covered by %s, the newest merge of %s in this revision: its message is not read, and it is ended with it", e.Key, e.Forge.Repository, e.Forge.Number, by.Key, e.Forge.Repository)
|
|
// Its pull request leaves its group's candidate as a merged message takes it out; nothing else of it is applied.
|
|
if err := ix.removeCandidate(ctx, e.Forge); err != nil {
|
|
ix.release(ctx, claimed)
|
|
return applied, fmt.Errorf("taking %s (%s#%d) out of its candidate: %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
|
|
}
|
|
continue
|
|
}
|
|
a, err := ix.ApplyMessage(ctx, e.Forge)
|
|
applied.Written = append(applied.Written, a.Written...)
|
|
if a.Late != "" {
|
|
applied.Late = a.Late
|
|
}
|
|
if errors.Is(err, ErrUnreadable) {
|
|
ix.logf("take-in: %s (%s#%d) can never be read, dropped: %v", e.Key, e.Forge.Repository, e.Forge.Number, err)
|
|
continue
|
|
}
|
|
if err != nil {
|
|
ix.release(ctx, claimed)
|
|
return applied, fmt.Errorf("taking %s (%s#%d) in: %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
|
|
}
|
|
}
|
|
for _, e := range claimed {
|
|
if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil {
|
|
ix.logf("take-in: %s.%s is taken in and not removed: %v", PendingState, e.Key, err)
|
|
}
|
|
}
|
|
ix.forget(claimed...)
|
|
ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s", len(claimed), own, byWords(by), claimed[0].Key)
|
|
for _, t := range saidWaits(claimed) {
|
|
t.By, t.Why = who, why
|
|
ix.sayTakeIn(ctx, t)
|
|
}
|
|
return applied, nil
|
|
}
|
|
|
|
func byWords(by string) string {
|
|
if by == "" {
|
|
return ""
|
|
}
|
|
return " (a hand act: " + by + ")"
|
|
}
|
|
|
|
// saidWaits are the taken events of every wait the entries were said under that is not ended yet.
|
|
func saidWaits(entries []PendingEntry) []takein.TakeIn {
|
|
by := map[string][]PendingEntry{}
|
|
for _, e := range entries {
|
|
if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending {
|
|
by[e.Value.Said] = append(by[e.Value.Said], e)
|
|
}
|
|
}
|
|
var out []takein.TakeIn
|
|
for _, key := range slices.Sorted(maps.Keys(by)) {
|
|
held := by[key]
|
|
if strings.HasPrefix(key, "g.") {
|
|
out = append(out, groupWait(takein.TakeInTaken, held[0].Value.Group, held))
|
|
continue
|
|
}
|
|
t := revisionWait(takein.TakeInTaken, held, held[0].Value.WaitsFor)
|
|
t.Key = key
|
|
out = append(out, t)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving
|
|
// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its
|
|
// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said
|
|
// pending under its oldest merge's key, and a wait it was said under before by another key is said taken.
|
|
func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error {
|
|
key, _ := delivery.ModulesKey(delivery.IndexerModule)
|
|
if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil {
|
|
return fmt.Errorf("the indexer's own module entry: %w", err)
|
|
}
|
|
if gk, err := graph.Key(delivery.IndexerModule); err == nil {
|
|
if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) {
|
|
return fmt.Errorf("the indexer's key of the module graph: %w", err)
|
|
}
|
|
}
|
|
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
|
|
if m, err := delivery.DecodeModule(value); err == nil && m.Manifest != "" {
|
|
if vk, err := delivery.VerdictsKey(m.Manifest); err == nil {
|
|
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
|
|
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
t := revisionWait(takein.TakeInPending, ready, names)
|
|
var ended, unmarked []PendingEntry
|
|
said, changed := false, false
|
|
for _, e := range ready {
|
|
switch {
|
|
case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending:
|
|
said = true
|
|
changed = changed || e.Value.WaitsFor != names
|
|
if e.Value.WaitsFor != names {
|
|
unmarked = append(unmarked, e)
|
|
}
|
|
case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending:
|
|
ended = append(ended, e)
|
|
unmarked = append(unmarked, e)
|
|
default:
|
|
unmarked = append(unmarked, e)
|
|
}
|
|
}
|
|
for _, old := range saidWaits(ended) {
|
|
ix.sayTakeIn(ctx, old)
|
|
}
|
|
if !said || changed {
|
|
ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own)
|
|
if !ix.sayTakeIn(ctx, t) {
|
|
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own })
|
|
return nil
|
|
}
|
|
unmarked = ready
|
|
}
|
|
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own })
|
|
return nil
|
|
}
|
|
|
|
// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it.
|
|
func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error {
|
|
if from == nil || to == nil {
|
|
return nil
|
|
}
|
|
value, deleted, ok := from.written(key)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return write(ctx, to, key, func(cur []byte, found bool) ([]byte, bool, error) {
|
|
switch {
|
|
case deleted:
|
|
return nil, found, nil
|
|
case found && string(cur) == string(value):
|
|
return nil, false, nil
|
|
}
|
|
return value, false, nil
|
|
})
|
|
}
|
|
|
|
// PendingWaits is what indexer_pending answers of the pending state's entries (ADR 0633 §5): the ready merges as one
|
|
// revision, each group being gathered, and the merges behind a group, each with the modules it holds back and the
|
|
// last take-in event said of it.
|
|
func PendingWaits(entries []PendingEntry) []takein.PendingWait {
|
|
w := WaitsOf(entries)
|
|
var out []takein.PendingWait
|
|
if len(w.Ready) > 0 {
|
|
out = append(out, waitOf(w.Ready, takein.TakeInRevision, ""))
|
|
}
|
|
for _, g := range slices.Sorted(maps.Keys(w.Groups)) {
|
|
out = append(out, waitOf(w.Groups[g], takein.TakeInGroup, ""))
|
|
}
|
|
for _, g := range slices.Sorted(maps.Keys(w.Behind)) {
|
|
out = append(out, waitOf(w.Behind[g], takein.TakeInRevision, g))
|
|
}
|
|
slices.SortStableFunc(out, func(a, b takein.PendingWait) int { return a.Since.Compare(b.Since) })
|
|
return out
|
|
}
|
|
|
|
func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
|
|
pw := takein.PendingWait{Key: entries[0].Key, What: what, Merges: mergesOf(entries), Behind: behind, Since: sinceOf(entries),
|
|
WaitsFor: entries[len(entries)-1].Value.WaitsFor, Holds: []string{}}
|
|
if what == takein.TakeInGroup {
|
|
pw.Group = entries[0].Value.Group
|
|
pw.Key = takein.GroupTakeInKey(pw.Group)
|
|
pw.Members = entries[len(entries)-1].Forge.Members
|
|
pw.WaitsFor = ""
|
|
}
|
|
for _, e := range entries {
|
|
pw.Keys = append(pw.Keys, e.Key)
|
|
for _, name := range e.Holds {
|
|
if !slices.Contains(pw.Holds, name) {
|
|
pw.Holds = append(pw.Holds, name)
|
|
}
|
|
}
|
|
if e.Value.Lost != "" {
|
|
pw.Lost = append(pw.Lost, e.Key)
|
|
}
|
|
if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) {
|
|
pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt
|
|
}
|
|
}
|
|
slices.Sort(pw.Holds)
|
|
return pw
|
|
}
|
|
|
|
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the
|
|
// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge,
|
|
// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are
|
|
// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after.
|
|
func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn {
|
|
r := TakenIn{Module: key}
|
|
if ix.Pending == nil {
|
|
r.Answer = "the pending state is not on the bus: nothing waits"
|
|
return r
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
r.Answer = err.Error()
|
|
return r
|
|
}
|
|
w := WaitsOf(entries)
|
|
var take []PendingEntry
|
|
switch {
|
|
case strings.HasPrefix(key, "g."):
|
|
for g, held := range w.Groups {
|
|
if takein.GroupTakeInKey(g) == key {
|
|
take = held
|
|
}
|
|
}
|
|
case len(w.Ready) > 0 && w.Ready[0].Key == key:
|
|
take = w.Ready
|
|
}
|
|
if len(take) == 0 {
|
|
r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key)
|
|
return r
|
|
}
|
|
take, lost, reasons := ix.loadNewest(ctx, take)
|
|
if len(lost) > 0 {
|
|
r.Answer = fmt.Sprintf("nothing is taken in: the message of %s is not read (%s); a newer merge of its repository covers it, indexer_drop_pending drops it", lost[0].Key, reasons[lost[0].Key])
|
|
return r
|
|
}
|
|
ix.logf("take-in: a hand take-in of %s (%d merges) asked by %s: %s", key, len(take), by, why)
|
|
hand := fmt.Sprintf("%s (%s)", by, why)
|
|
if _, err := ix.takeInWholeBy(ctx, take, own, hand, by, why); err != nil {
|
|
r.Answer = err.Error()
|
|
return r
|
|
}
|
|
r.Stored = true
|
|
r.Answer = fmt.Sprintf("%d merges taken in by the running indexer %s", len(take), own)
|
|
if _, _, err := ix.evaluate(ctx, own, false); err != nil {
|
|
r.Answer += "; what waited behind it is not evaluated: " + err.Error()
|
|
}
|
|
return r
|
|
}
|
|
|
|
// DropPending is the hand drop (indexer_drop_pending, ADR 0633 §5): the one pending merge under key, abandoned
|
|
// because a later merge replaced it, is removed and nothing of it is taken in. Its wait is said taken, naming the drop;
|
|
// the merges left under that wait are said pending again by the evaluation that follows.
|
|
func (ix *Indexer) DropPending(ctx context.Context, key, by, why string) TakenIn {
|
|
r := TakenIn{Module: key}
|
|
if ix.Pending == nil {
|
|
r.Answer = "the pending state is not on the bus: nothing waits"
|
|
return r
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
r.Answer = err.Error()
|
|
return r
|
|
}
|
|
at := slices.IndexFunc(entries, func(e PendingEntry) bool { return e.Key == key })
|
|
if at < 0 {
|
|
r.Answer = fmt.Sprintf("%s holds no merge under %s", PendingState, key)
|
|
return r
|
|
}
|
|
e := entries[at]
|
|
if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil {
|
|
r.Answer = fmt.Sprintf("%s.%s is not removed: %v", PendingState, key, err)
|
|
return r
|
|
}
|
|
ix.forget(e)
|
|
ix.logf("take-in: %s (%s#%d at %s) dropped by hand, asked by %s: %s; nothing of it is taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge, by, why)
|
|
if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending {
|
|
var same []PendingEntry
|
|
for _, o := range entries {
|
|
if o.Key != key && o.Value.Said == e.Value.Said {
|
|
same = append(same, o)
|
|
}
|
|
}
|
|
for _, t := range saidWaits([]PendingEntry{e}) {
|
|
t.By, t.Why, t.Dropped = by, why, true
|
|
ix.sayTakeIn(ctx, t)
|
|
}
|
|
ix.markSaid(ctx, same, nil, func(p *Pending) { p.Said, p.SaidKind = "", "" })
|
|
}
|
|
r.Stored, r.Answer = true, fmt.Sprintf("%s dropped: %s#%d at %s is not taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge)
|
|
if own != "" {
|
|
if _, _, err := ix.evaluate(ctx, own, false); err != nil {
|
|
r.Answer += "; what waits is not evaluated again: " + err.Error()
|
|
}
|
|
}
|
|
return r
|
|
}
|
|
|
|
// TakeInMerge is the hand take-in of one merge (indexer_take_in_merge, issue 1585): its merged message is asked of the forge
|
|
// bridge and handed to the take-in as a delivered merged message is (Arrive, under the stream sequence seq, which keeps the
|
|
// merge order), so it is held for the indexer its revision names or taken in, and a merge already held under that
|
|
// sequence is not written twice. For a merge the take-in never took in. The bridge's own words answer when it cannot give
|
|
// the message.
|
|
func (ix *Indexer) TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) TakenIn {
|
|
r := TakenIn{Module: takein.PendingKey(seq)}
|
|
if ix.Message == nil {
|
|
r.Answer = "this indexer does not ask the forge bridge"
|
|
return r
|
|
}
|
|
data, err := ix.Message(ctx, repository, merge)
|
|
if err != nil {
|
|
r.Answer = fmt.Sprintf("the forge bridge could not answer for the merge of %s at %s: %v", repository, merge, err)
|
|
return r
|
|
}
|
|
f, err := forgemsg.DecodeForge(data)
|
|
if err != nil {
|
|
r.Answer = fmt.Sprintf("the forge bridge's answer for %s at %s is not a forge message: %v", repository, merge, err)
|
|
return r
|
|
}
|
|
if f.Kind != forgemsg.ForgeMerged || f.Merge != merge || f.Repository != repository {
|
|
r.Answer = fmt.Sprintf("the forge bridge's answer is not the merge of %s at %s", repository, merge)
|
|
return r
|
|
}
|
|
ix.logf("take-in: a hand take-in of the merge %s#%d at %s (sequence %d) asked by %s: %s", f.Repository, f.Number, f.Merge, seq, by, why)
|
|
a, words, err := ix.Arrive(ctx, seq, f)
|
|
if err != nil {
|
|
r.Answer = err.Error()
|
|
return r
|
|
}
|
|
r.Stored, r.Repository, r.Commit = true, f.Repository, f.Merge
|
|
r.Answer = fmt.Sprintf("%s#%d at %s handed to the take-in as a delivered merge (%d keys written)", f.Repository, f.Number, f.Merge, len(a.Written))
|
|
if words != "" {
|
|
r.Answer += "; " + words
|
|
}
|
|
return r
|
|
}
|