A merge waiting behind a gathered group is named by indexer_pending under its own key; when the group ends it joins the group's revision under the oldest member's key, so its own key was never said taken. takenWaits now also says every claimed entry's key, no key twice. T8 checks the merge behind the group ends under its own key. Builds: design 47 "Taken in by the indexer it names" (A wait is bounded, said and recoverable); 1966/AC195-1, AC195-3 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
1480 lines
61 KiB
Go
1480 lines
61 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
|
|
|
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
|
"git.novox.be/novox/mesh-delivery/sdk/trunk"
|
|
"git.novox.be/novox/mesh-sdk/go/busretry"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
|
|
"mesh-delivery-indexer/judge"
|
|
|
|
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
|
)
|
|
|
|
// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is
|
|
// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another
|
|
// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it.
|
|
const (
|
|
// PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed
|
|
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
|
|
// 0595 §11), history 1, reached by the indexer alone.
|
|
PendingState = "pending"
|
|
PendingBucket = Module + "_" + PendingState
|
|
// PendingTool reads it (ADR 0633 §5); TakeInPendingTool and DropPendingTool steer it.
|
|
PendingTool = "indexer_pending"
|
|
TakeInPendingTool = "indexer_take_in_pending"
|
|
DropPendingTool = "indexer_drop_pending"
|
|
)
|
|
|
|
// errClaimLost says another take-in claimed a pending merge first: this one took nothing in.
|
|
var errClaimLost = errors.New("another take-in claimed the merges first, so this one took nothing in")
|
|
|
|
// NoBuild opens what a revision waits for when it gives the indexer no fingerprint: the reason follows. No indexer
|
|
// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending).
|
|
const NoBuild = "no build: "
|
|
|
|
// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1):
|
|
// the message is left unacknowledged, so the bus delivers it again, and nothing is written.
|
|
var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known")
|
|
|
|
// PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many
|
|
// modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out
|
|
// from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it.
|
|
type PendingRef struct {
|
|
Repository string `json:"repository"`
|
|
Number int `json:"number,omitempty"`
|
|
Merge string `json:"merge"`
|
|
// Branch is the pull request's branch, which names its candidate: a merge covered by a newer one is taken out of it.
|
|
Branch string `json:"branch,omitempty"`
|
|
Members []meshdeliveryclient.Head `json:"members,omitempty"`
|
|
// Holds are the modules the merge's manifests name, which indexer_pending says it holds back.
|
|
Holds []string `json:"holds,omitempty"`
|
|
}
|
|
|
|
// Pending is one value of the pending state: a reference to the merged message (Seq, its sequence on the forge bridge's
|
|
// stream, and Ref), or, in a value an earlier indexer wrote, the message whole as the forge bridge encoded it; the group it
|
|
// was merged in (Ended once the group's end was heard), since when it is held, the indexer build its revision waits
|
|
// for (absent while it waits for nothing yet, or behind a group being gathered), the build of the indexer that last
|
|
// wrote it, the wait it was last said under on the take-in stream (Said, SaidAt), and Taking, set by a take-in that
|
|
// claimed it: a second indexer that finds it claimed takes nothing in (ADR 0633 §1: never two take-ins of one merge).
|
|
type Pending struct {
|
|
V int `json:"v"`
|
|
Seq uint64 `json:"seq,omitempty"`
|
|
Ref *PendingRef `json:"ref,omitempty"`
|
|
Forge json.RawMessage `json:"forge,omitempty"`
|
|
Lost string `json:"lost,omitempty"`
|
|
Refused string `json:"refused,omitempty"`
|
|
Group string `json:"group,omitempty"`
|
|
Ended bool `json:"ended,omitempty"`
|
|
Since time.Time `json:"since"`
|
|
WaitsFor string `json:"waits-for,omitempty"`
|
|
LastWriter string `json:"last-writer"`
|
|
Said string `json:"said,omitempty"`
|
|
SaidKind string `json:"said-kind,omitempty"`
|
|
SaidAt time.Time `json:"said-at,omitzero"`
|
|
Taking string `json:"taking,omitempty"`
|
|
// Unread is why the merged message could not be read for a transport fault (the forge bridge restarting, a deadline;
|
|
// ADR 0664's classes): not lost, it is read again on the next message heard on the forge, at the start and by
|
|
// indexer_take_in_pending (issue 1966 note 153, ADR 0668 §1).
|
|
Unread string `json:"unread,omitempty"`
|
|
// UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge
|
|
// heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3).
|
|
UnreadSince time.Time `json:"unread-since,omitzero"`
|
|
}
|
|
|
|
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
|
|
// value that keeps a reference, Forge holds only the fields of the reference (Light) until Indexer.load reads the message
|
|
// from the stream; Holds are the modules the merge's manifests name.
|
|
type PendingEntry struct {
|
|
Key string
|
|
Revision uint64
|
|
Value Pending
|
|
Forge forgemsg.Forge
|
|
Light bool
|
|
Holds []string
|
|
// Covered says its message is not read or applied: either a newer merge of the same repository is in the same revision
|
|
// (applying the newest writes every module whose manifest differs from what is held, modules.go), or its repository's
|
|
// taken merge is at or ahead of it (loadNewest, takenAlready), so applying it would write nothing.
|
|
Covered bool
|
|
}
|
|
|
|
// DecodePending reads one pending value and its merged message.
|
|
func DecodePending(data []byte) (Pending, forgemsg.Forge, error) {
|
|
var p Pending
|
|
if err := json.Unmarshal(data, &p); err != nil {
|
|
return p, forgemsg.Forge{}, fmt.Errorf("not a pending value: %w", err)
|
|
}
|
|
if p.V != 1 {
|
|
return p, forgemsg.Forge{}, fmt.Errorf("a pending value of version %d, not 1", p.V)
|
|
}
|
|
if len(p.Forge) == 0 && p.Ref != nil {
|
|
r := p.Ref
|
|
return p, forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: r.Repository, Number: r.Number,
|
|
Merge: r.Merge, Branch: r.Branch, Group: p.Group, Members: r.Members}, nil
|
|
}
|
|
f, err := forgemsg.DecodeForge(p.Forge)
|
|
if err != nil {
|
|
return p, f, fmt.Errorf("its merged message: %w", err)
|
|
}
|
|
if f.Kind != forgemsg.ForgeMerged {
|
|
return p, f, fmt.Errorf("a pending value holds a %s message, not a merged one", f.Kind)
|
|
}
|
|
return p, f, nil
|
|
}
|
|
|
|
func encodePending(p Pending) ([]byte, error) {
|
|
p.V = 1
|
|
return json.Marshal(p)
|
|
}
|
|
|
|
// ReadPending is every value the pending state holds, in key order (the merge order); a key that does not read is
|
|
// said through logf and left.
|
|
func ReadPending(ctx context.Context, st Store, logf func(string, ...any)) ([]PendingEntry, error) {
|
|
keys, err := st.Keys(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("listing %s: %w", PendingState, err)
|
|
}
|
|
slices.Sort(keys)
|
|
var out []PendingEntry
|
|
for _, key := range keys {
|
|
value, rev, found, err := st.Get(ctx, key)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s.%s: %w", PendingState, key, err)
|
|
}
|
|
if !found {
|
|
continue
|
|
}
|
|
p, f, err := DecodePending(value)
|
|
if err != nil {
|
|
if logf != nil {
|
|
logf("%s.%s is left: %v", PendingState, key, err)
|
|
}
|
|
continue
|
|
}
|
|
e := PendingEntry{Key: key, Revision: rev, Value: p, Forge: f, Light: len(p.Forge) == 0}
|
|
if p.Ref != nil {
|
|
e.Holds = p.Ref.Holds
|
|
} else {
|
|
e.Holds = holdsOf(f)
|
|
}
|
|
out = append(out, e)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// holdsOf are the modules a merged message's manifests name, sorted.
|
|
func holdsOf(f forgemsg.Forge) []string {
|
|
var out []string
|
|
for _, raw := range f.Manifests {
|
|
if name := manifestModule(raw); name != "" && !slices.Contains(out, name) {
|
|
out = append(out, name)
|
|
}
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// refOf is what a pending value keeps of the merged message f (PendingRef).
|
|
func refOf(f forgemsg.Forge) *PendingRef {
|
|
return &PendingRef{Repository: f.Repository, Number: f.Number, Merge: f.Merge, Branch: f.Branch, Members: f.Members, Holds: holdsOf(f)}
|
|
}
|
|
|
|
// heldMessage is the message Arrive was handed for a sequence, kept in memory until its merge is taken in or dropped.
|
|
func (ix *Indexer) heldMessage(seq uint64) (forgemsg.Forge, bool) {
|
|
ix.heldMu.Lock()
|
|
defer ix.heldMu.Unlock()
|
|
f, ok := ix.held[seq]
|
|
return f, ok
|
|
}
|
|
|
|
func (ix *Indexer) hold(seq uint64, f forgemsg.Forge) {
|
|
ix.heldMu.Lock()
|
|
defer ix.heldMu.Unlock()
|
|
if ix.held == nil {
|
|
ix.held = map[uint64]forgemsg.Forge{}
|
|
}
|
|
ix.held[seq] = f
|
|
}
|
|
|
|
func (ix *Indexer) forget(entries ...PendingEntry) {
|
|
ix.heldMu.Lock()
|
|
defer ix.heldMu.Unlock()
|
|
for _, e := range entries {
|
|
delete(ix.held, e.Value.Seq)
|
|
}
|
|
}
|
|
|
|
// load gives the entry the merged message whole: the copy an earlier value holds, the message Arrive was handed (kept in
|
|
// memory), else the message read from the forge bridge's stream by the value's sequence. A stream that no longer holds it
|
|
// answers ErrForgeGone, a bus that refuses the read its error; a message that is not the merge the value refers to (a
|
|
// stream rebuilt under the same sequence) is refused.
|
|
func (ix *Indexer) load(ctx context.Context, e PendingEntry) (PendingEntry, error) {
|
|
if !e.Light {
|
|
return e, nil
|
|
}
|
|
if f, ok := ix.heldMessage(e.Value.Seq); ok {
|
|
e.Forge, e.Light = f, false
|
|
return e, nil
|
|
}
|
|
if ix.Message == nil {
|
|
return e, fmt.Errorf("the forge bridge is not asked by this indexer, so %s (%s#%d) cannot be read", e.Key, e.Forge.Repository, e.Forge.Number)
|
|
}
|
|
data, err := ix.Message(ctx, e.Forge.Repository, e.Forge.Merge, e.Forge.Number)
|
|
if err != nil {
|
|
return e, fmt.Errorf("the forge bridge could not answer for the merge of %s#%d at %s: %w", e.Forge.Repository, e.Forge.Number, e.Forge.Merge, err)
|
|
}
|
|
f, err := forgemsg.DecodeForge(data)
|
|
if err != nil {
|
|
return e, fmt.Errorf("the forge bridge's answer for %s is not a forge message: %w", e.Key, err)
|
|
}
|
|
if f.Kind != forgemsg.ForgeMerged || f.Repository != e.Forge.Repository || f.Merge != e.Forge.Merge || (e.Forge.Number > 0 && f.Number != e.Forge.Number) {
|
|
return e, fmt.Errorf("the forge bridge's answer is not the merge %s#%d at %s that %s refers to", e.Forge.Repository, e.Forge.Number, e.Forge.Merge, e.Key)
|
|
}
|
|
e.Forge, e.Light = f, false
|
|
return e, nil
|
|
}
|
|
|
|
// loadNewest prepares the ready merges of one revision, in merge order. Of the merges of one repository only the newest is
|
|
// loaded: applying it writes every module whose manifest differs from what is held, so it covers the older ones, which are
|
|
// marked Covered and never read. A newest merge the forge bridge cannot give is answered in lost with its reason, by key, and its revision is held (holdLost).
|
|
func (ix *Indexer) loadNewest(ctx context.Context, entries []PendingEntry) (out, lost []PendingEntry, reasons map[string]string, unread map[string]bool) {
|
|
out = slices.Clone(entries)
|
|
reasons, unread = map[string]string{}, map[string]bool{}
|
|
newest := map[string]bool{}
|
|
for i := len(out) - 1; i >= 0; i-- {
|
|
repo := strings.ToLower(out[i].Forge.Repository)
|
|
if newest[repo] {
|
|
out[i].Covered = true
|
|
continue
|
|
}
|
|
newest[repo] = true
|
|
// A merge at or behind its repository's taken merge writes nothing: it is ended as covered without reading its
|
|
// message, which a bridge that cannot give it would otherwise hold the whole revision for (issue 1966). A merge whose
|
|
// ancestry cannot be asked is read as before.
|
|
if taken, err := ix.takenAlready(ctx, forgemsg.GroupMerge{Repository: out[i].Forge.Repository, Number: out[i].Forge.Number, Merge: out[i].Forge.Merge}); err == nil && taken {
|
|
ix.logf("take-in: %s (%s#%d) is covered by the taken merge of %s, its message is not read", out[i].Key, out[i].Forge.Repository, out[i].Forge.Number, out[i].Forge.Repository)
|
|
out[i].Covered = true
|
|
continue
|
|
}
|
|
full, err := ix.load(ctx, out[i])
|
|
if err != nil {
|
|
lost = append(lost, out[i])
|
|
reasons[out[i].Key] = err.Error()
|
|
// A transport fault is not the bridge's answer: the message is unread, never lost (ADR 0668 §1).
|
|
unread[out[i].Key] = busretry.Undelivered(err) || busretry.TimedOut(err)
|
|
continue
|
|
}
|
|
out[i] = full
|
|
}
|
|
slices.Reverse(lost)
|
|
return out, lost, reasons, unread
|
|
}
|
|
|
|
// Waits is the pending state read as what waits (ADR 0633 §3, §4): Ready are the merges no group still being gathered
|
|
// holds back, in merge order, which are one revision together; Groups are the groups still being gathered, each with
|
|
// its members held so far; Behind are the merges in a gathered group's member repositories that wait behind it, by the
|
|
// group's take-in key.
|
|
type Waits struct {
|
|
Ready []PendingEntry
|
|
Groups map[string][]PendingEntry
|
|
Behind map[string][]PendingEntry
|
|
}
|
|
|
|
// WaitsOf works out what waits from the entries, in key order. A member of a group whose end was not heard is
|
|
// gathered; from the first such member on, a later merge in one of the group's members' repositories waits behind it,
|
|
// because its tree holds that member's commit. Every other merge is ready.
|
|
func WaitsOf(entries []PendingEntry) Waits {
|
|
w := Waits{Groups: map[string][]PendingEntry{}, Behind: map[string][]PendingEntry{}}
|
|
blocked := map[string]string{} // repository, lower case, to the group key it waits behind
|
|
for _, e := range entries {
|
|
if e.Value.Group != "" && !e.Value.Ended {
|
|
w.Groups[e.Value.Group] = append(w.Groups[e.Value.Group], e)
|
|
for _, m := range e.Forge.Members {
|
|
if _, ok := blocked[strings.ToLower(m.Repository)]; !ok {
|
|
blocked[strings.ToLower(m.Repository)] = takein.GroupTakeInKey(e.Value.Group)
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if g, ok := blocked[strings.ToLower(e.Forge.Repository)]; ok {
|
|
w.Behind[g] = append(w.Behind[g], e)
|
|
continue
|
|
}
|
|
w.Ready = append(w.Ready, e)
|
|
}
|
|
return w
|
|
}
|
|
|
|
// mergesOf are the merges of entries, oldest first.
|
|
func mergesOf(entries []PendingEntry) []takein.Merge {
|
|
out := make([]takein.Merge, len(entries))
|
|
for i, e := range entries {
|
|
out[i] = takein.Merge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func sinceOf(entries []PendingEntry) time.Time {
|
|
var at time.Time
|
|
for _, e := range entries {
|
|
if at.IsZero() || e.Value.Since.Before(at) {
|
|
at = e.Value.Since
|
|
}
|
|
}
|
|
return at
|
|
}
|
|
|
|
// groupWait is the take-in body of a group being gathered.
|
|
func groupWait(kind, group string, held []PendingEntry) takein.TakeIn {
|
|
t := takein.TakeIn{Kind: kind, Key: takein.GroupTakeInKey(group), What: takein.TakeInGroup, Group: group,
|
|
Merges: mergesOf(held), Since: sinceOf(held)}
|
|
if len(held) > 0 {
|
|
t.Members = held[len(held)-1].Forge.Members
|
|
}
|
|
return t
|
|
}
|
|
|
|
// revisionWait is the take-in body of a revision: its key is its oldest merge's.
|
|
func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.TakeIn {
|
|
return takein.TakeIn{Kind: kind, Key: ready[0].Key, What: takein.TakeInRevision, Merges: mergesOf(ready),
|
|
WaitsFor: waitsFor, Since: sinceOf(ready)}
|
|
}
|
|
|
|
// pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears
|
|
// (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read
|
|
// that fails otherwise counts as on the bus: the step that follows answers the error.
|
|
// errVerdictsNotYet is a take-in held because the verdicts state is not on the bus yet (takeInWholeBy).
|
|
var errVerdictsNotYet = errors.New("the verdicts state is not on the bus yet: nothing is taken in until it is")
|
|
|
|
// verdictsOnBus says the verdicts state is on the bus (a take-in judges what it writes there).
|
|
func (ix *Indexer) verdictsOnBus(ctx context.Context) bool {
|
|
_, _, _, err := ix.Verdicts.Get(ctx, "on-the-bus")
|
|
return !errors.Is(err, process.ErrNotYet)
|
|
}
|
|
|
|
func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
|
|
_, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0))
|
|
return !errors.Is(err, process.ErrNotYet)
|
|
}
|
|
|
|
// ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again.
|
|
func (ix *Indexer) ownBuildNow(context.Context) string {
|
|
ix.ownMu.Lock()
|
|
defer ix.ownMu.Unlock()
|
|
return ix.ownFP
|
|
}
|
|
|
|
// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a
|
|
// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to
|
|
// pending; the stream delivers it again to the process that knows its build.
|
|
func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) {
|
|
ix.ownMu.Lock()
|
|
since, why := ix.ownSince, ix.ownWhy
|
|
ix.ownMu.Unlock()
|
|
if since.IsZero() { // never fixed: the start did not run (no engine wired)
|
|
since, why = ix.now(), "the build this program runs was never asked"
|
|
}
|
|
ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision,
|
|
Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why})
|
|
}
|
|
|
|
// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of
|
|
// each wait from the state. A publish that fails is said in the log: the next evaluation says it again.
|
|
func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool {
|
|
if ix.SayTakeIn == nil {
|
|
return true
|
|
}
|
|
if err := ix.SayTakeIn(ctx, t); err != nil {
|
|
ix.logf("take-in: %s.%s is not said: %v", t.Kind, t.Key, err)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// markSaid writes on each entry the wait it was said under, by compare-and-set; a lost one is left (its next
|
|
// evaluation says it again).
|
|
func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *takein.TakeIn, change func(*Pending)) {
|
|
for _, e := range entries {
|
|
p := e.Value
|
|
if t != nil {
|
|
p.Said, p.SaidKind, p.SaidAt = t.Key, t.Kind, ix.now()
|
|
}
|
|
if change != nil {
|
|
change(&p)
|
|
}
|
|
data, err := encodePending(p)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil {
|
|
ix.logf("take-in: %s.%s is not written again: %v", PendingState, e.Key, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Arrive takes one merged message of the forge bridge's stream in under ADR 0633: seq is its stream sequence. It is
|
|
// written to pending first (a redelivered message finds its key there and writes no second), and then what pending
|
|
// holds is evaluated (Evaluate): taken in whole when its revision names the build this program runs, or held for the
|
|
// indexer it names. It answers what was taken in, and a line saying what waits. A message is never written while the
|
|
// build this program runs is not known (ErrOwnBuildUnknown).
|
|
func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Applied, string, error) {
|
|
if ix.Pending == nil || ix.Modules == nil || !ix.pendingOnBus(ctx) {
|
|
// The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before,
|
|
// whether or not the build this program runs is known.
|
|
a, err := ix.ApplyMessage(ctx, f)
|
|
return a, "", err
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
if own == "" {
|
|
ix.sayOwnBuildUnknown(ctx, seq, f)
|
|
return Applied{}, "", ErrOwnBuildUnknown
|
|
}
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
key := takein.PendingKey(seq)
|
|
ix.hold(seq, f)
|
|
data, err := encodePending(Pending{Seq: seq, Ref: refOf(f), Group: f.Group, Since: ix.now(), LastWriter: own})
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
switch err := ix.Pending.Create(ctx, key, data); {
|
|
case errors.Is(err, process.ErrNotYet):
|
|
// The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before.
|
|
ix.logf("take-in: the state %s is not on the bus yet, so %s#%d is taken in at once, as before ADR 0633", PendingBucket, f.Repository, f.Number)
|
|
a, err := ix.ApplyMessage(ctx, f)
|
|
return a, "", err
|
|
case err != nil && !errors.Is(err, ErrConflict):
|
|
return Applied{}, "", fmt.Errorf("%s.%s: %w", PendingState, key, err)
|
|
}
|
|
if f.Group != "" {
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
if held := WaitsOf(entries).Groups[f.Group]; len(held) > 0 {
|
|
t := groupWait(takein.TakeInPending, f.Group, held)
|
|
if ix.sayTakeIn(ctx, t) {
|
|
ix.markSaid(ctx, held, &t, nil)
|
|
}
|
|
}
|
|
}
|
|
return ix.evaluate(ctx, own, false)
|
|
}
|
|
|
|
// EndGroup takes a group's end in (ADR 0633 §4): once every member it names is held, the group's members are marked
|
|
// ended, the group's wait is said taken (take-in.taken under its group key), and pending is evaluated, the group now
|
|
// one revision. A member named and neither held nor taken in yet answers an error, so the end is delivered again once
|
|
// the member arrived. An end of a group taken in already (every member's merge at or behind its repository's taken
|
|
// merge) changes nothing and is said: it is never delivered again for ever.
|
|
func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Applied, string, error) {
|
|
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
|
return Applied{}, "no pending state: the group's merges were taken in one by one", nil
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
if own == "" {
|
|
return Applied{}, "", ErrOwnBuildUnknown
|
|
}
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if errors.Is(err, process.ErrNotYet) {
|
|
return Applied{}, "the pending state is not on the bus yet: the group's merges were taken in one by one", nil
|
|
}
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
var held []PendingEntry
|
|
for _, e := range entries {
|
|
if e.Value.Group == end.Group {
|
|
held = append(held, e)
|
|
}
|
|
}
|
|
isHeld := func(m forgemsg.GroupMerge) bool {
|
|
return slices.ContainsFunc(held, func(e PendingEntry) bool {
|
|
return trunk.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number
|
|
})
|
|
}
|
|
var missing []string
|
|
for _, m := range end.Members {
|
|
if isHeld(m) {
|
|
continue
|
|
}
|
|
taken, err := ix.takenAlready(ctx, m)
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
if !taken {
|
|
missing = append(missing, fmt.Sprintf("%s#%d", m.Repository, m.Number))
|
|
}
|
|
}
|
|
if len(missing) > 0 {
|
|
return Applied{}, "", fmt.Errorf("the end of the group %q names %s, not held yet: it waits for their merged messages", end.Group, strings.Join(missing, ", "))
|
|
}
|
|
if len(held) == 0 {
|
|
ix.logf("take-in: the end of the group %q comes after its take-in: every member is taken in already, nothing changes", end.Group)
|
|
return Applied{}, "the group is taken in already", nil
|
|
}
|
|
if !slices.ContainsFunc(held, func(e PendingEntry) bool { return !e.Value.Ended }) {
|
|
return ix.evaluate(ctx, own, false)
|
|
}
|
|
// The group's wait ends first (take-in.taken under its group key); a revision it then is, pending for another
|
|
// indexer, is said pending under its oldest merge's key by the evaluation that follows.
|
|
t := groupWait(takein.TakeInTaken, end.Group, held)
|
|
said := &t
|
|
if !ix.sayTakeIn(ctx, t) {
|
|
said = nil // said again by the take-in or the hold that follows (saidWaits)
|
|
}
|
|
ix.markSaid(ctx, held, said, func(p *Pending) { p.Ended = true })
|
|
ix.logf("take-in: the group %q ended with %d members held: one revision now", end.Group, len(held))
|
|
return ix.evaluate(ctx, own, false)
|
|
}
|
|
|
|
// endTakenGroups ends every group wait whose members are all taken in (issue 1585, notes 70 and 71): a group's end that
|
|
// never came, or a member taken in by hand (indexer_take_in_merge, a m. key), would hold the group and every merge behind
|
|
// it for ever. A member held is taken in when its merge is at or behind its repository's taken merge; a member no longer
|
|
// held when its head is in that taken merge. The wait is said taken (take-in.taken under the group key), and only then are
|
|
// its keys removed: a wait that could not be said stays and is ended at the next evaluation. It answers whether one ended.
|
|
func (ix *Indexer) endTakenGroups(ctx context.Context, entries []PendingEntry) (bool, error) {
|
|
if ix.Taken == nil {
|
|
return false, nil
|
|
}
|
|
w := WaitsOf(entries)
|
|
ended := false
|
|
for _, group := range slices.Sorted(maps.Keys(w.Groups)) {
|
|
held := w.Groups[group]
|
|
all, err := ix.groupTaken(ctx, held)
|
|
if err != nil {
|
|
// Best effort: a probe that fails keeps this group's wait and stops nothing else; the next evaluation asks again.
|
|
ix.logf("take-in: whether the group %q is taken in cannot be read, so its wait is kept: %v", group, err)
|
|
continue
|
|
}
|
|
if !all {
|
|
continue
|
|
}
|
|
if !ix.sayTakeIn(ctx, groupWait(takein.TakeInTaken, group, held)) {
|
|
continue
|
|
}
|
|
for _, e := range held {
|
|
if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil {
|
|
ix.logf("take-in: %s.%s belongs to a group taken in and is not removed: %v", PendingState, e.Key, err)
|
|
}
|
|
}
|
|
ix.forget(held...)
|
|
ix.logf("take-in: the group %q ended: every member is taken in already (%d merges held, removed)", group, len(held))
|
|
ended = true
|
|
}
|
|
return ended, nil
|
|
}
|
|
|
|
// groupTaken says whether every member of the group whose entries are held is taken in.
|
|
func (ix *Indexer) groupTaken(ctx context.Context, held []PendingEntry) (bool, error) {
|
|
isHeld := func(m meshdeliveryclient.Head) bool {
|
|
return slices.ContainsFunc(held, func(e PendingEntry) bool {
|
|
return trunk.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number
|
|
})
|
|
}
|
|
for _, e := range held {
|
|
if ok, err := ix.takenAlready(ctx, forgemsg.GroupMerge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge}); err != nil || !ok {
|
|
return false, err
|
|
}
|
|
}
|
|
for _, m := range held[len(held)-1].Forge.Members {
|
|
if isHeld(m) {
|
|
continue
|
|
}
|
|
ok, err := ix.headTaken(ctx, m)
|
|
if err != nil || !ok {
|
|
return false, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// headTaken says whether a group member that is not held was taken in: its head is the taken merge of its repository or
|
|
// in it.
|
|
func (ix *Indexer) headTaken(ctx context.Context, m meshdeliveryclient.Head) (bool, error) {
|
|
key, err := trunk.RepositoryKey(m.Repository)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
value, _, found, err := ix.Taken.Get(ctx, key)
|
|
if errors.Is(err, process.ErrNotYet) || (err == nil && !found) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, fmt.Errorf("taken.%s: %w", key, err)
|
|
}
|
|
cur, err := takein.DecodeTaken(value)
|
|
if err != nil || cur.Merge == "" || ix.Contains == nil {
|
|
return false, nil
|
|
}
|
|
if cur.Merge == m.Head {
|
|
return true, nil
|
|
}
|
|
in, err := ix.Contains(ctx, m.Repository, m.Head, cur.Merge)
|
|
if notFound(err) {
|
|
return false, nil // the forge does not have it (a rewritten history?): not known to be taken, as older has it
|
|
}
|
|
return in, err
|
|
}
|
|
|
|
// takenAlready says whether a merge (a group member's, or the newest pending merge of a repository in loadNewest) is at or
|
|
// behind its repository's taken merge.
|
|
func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (bool, error) {
|
|
if ix.Taken == nil {
|
|
return false, nil
|
|
}
|
|
key, err := trunk.RepositoryKey(m.Repository)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
value, _, found, err := ix.Taken.Get(ctx, key)
|
|
if errors.Is(err, process.ErrNotYet) || (err == nil && !found) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, fmt.Errorf("taken.%s: %w", key, err)
|
|
}
|
|
cur, err := takein.DecodeTaken(value)
|
|
if err != nil || cur.Merge == "" {
|
|
return false, nil
|
|
}
|
|
if cur.Merge == m.Merge {
|
|
return true, nil
|
|
}
|
|
return ix.older(ctx, m.Repository, m.Merge, cur.Merge)
|
|
}
|
|
|
|
// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts:
|
|
// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a
|
|
// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an
|
|
// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for
|
|
// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the
|
|
// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer.
|
|
func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
|
|
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
|
// Nothing can wait in a state not on the bus: the start asks no build and does not wait for one.
|
|
return Applied{}, "the pending state is not on the bus yet: nothing waits", nil
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
if own == "" {
|
|
ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again")
|
|
return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil
|
|
}
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
a, words, err := ix.evaluate(ctx, own, true)
|
|
if errors.Is(err, process.ErrNotYet) {
|
|
return a, "the pending state is not on the bus yet: nothing waits", nil
|
|
}
|
|
if err == nil && len(a.Written) == 0 {
|
|
// A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says).
|
|
ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words)
|
|
}
|
|
return a, words, err
|
|
}
|
|
|
|
// UnreadAgain evaluates pending again when a merge in it is unread for a transport fault (Pending.Unread): told after
|
|
// every message heard on the forge, of any kind, so a merge whose message the forge bridge could not give while it
|
|
// restarted is read again once the forge is heard again, with no timer (issue 1966 note 153, ADR 0668 §4). It reads
|
|
// pending once and does nothing when no merge is unread.
|
|
func (ix *Indexer) UnreadAgain(ctx context.Context) error {
|
|
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
|
return nil
|
|
}
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Unread != "" }) {
|
|
if err := ix.PendingAgain(ctx); err != nil {
|
|
return err
|
|
}
|
|
if entries, err = ReadPending(ctx, ix.Pending, ix.logf); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// The forge was heard, so the bridge that says it is: a read still failing past the bound is a finding (ADR 0668 §3).
|
|
ix.unreadFindings(ctx, entries)
|
|
return nil
|
|
}
|
|
|
|
// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report
|
|
// (ADR 0633 §1, §3), or the verdicts state bound (issue 1966): a revision that now names the running build is taken in, and one that names another build is held
|
|
// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known.
|
|
func (ix *Indexer) PendingAgain(ctx context.Context) error {
|
|
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
|
return nil
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
if own == "" {
|
|
return nil
|
|
}
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
_, words, err := ix.evaluate(ctx, own, false)
|
|
if errors.Is(err, process.ErrNotYet) {
|
|
return nil
|
|
}
|
|
if err == nil && words != "" {
|
|
ix.logf("take-in: evaluated again: %s", words)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// evaluate works out what pending's ready merges name, together (ADR 0633 §3), and takes them in or holds them.
|
|
func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applied, string, error) {
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
// A group wait whose members are all taken in ends first, so what waited behind it is evaluated in this same call.
|
|
if ended, err := ix.endTakenGroups(ctx, entries); err != nil {
|
|
return Applied{}, "", err
|
|
} else if ended {
|
|
if entries, err = ReadPending(ctx, ix.Pending, ix.logf); err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
}
|
|
w := WaitsOf(entries)
|
|
if len(w.Ready) == 0 {
|
|
return Applied{}, ix.waitingLine(w), nil
|
|
}
|
|
if !start {
|
|
for _, e := range w.Ready {
|
|
if e.Value.Taking != "" {
|
|
ix.logf("take-in: %s.%s is claimed by the take-in %s, so this indexer takes nothing in", PendingState, e.Key, e.Value.Taking)
|
|
return Applied{}, "claimed by another take-in", nil
|
|
}
|
|
}
|
|
}
|
|
// Of each repository's merges only the newest is read (from memory, else asked of the forge bridge: a pending value
|
|
// keeps a reference); the older ones are covered by it. A newest merge that cannot be read is said and
|
|
// holds its revision: nothing of it is taken in silently.
|
|
ready, lost, reasons, unread := ix.loadNewest(ctx, w.Ready)
|
|
if len(lost) > 0 {
|
|
return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread)
|
|
}
|
|
names, dry, err := ix.computeOver(ctx, ready)
|
|
if err != nil {
|
|
return Applied{}, "", err
|
|
}
|
|
if names == own {
|
|
a, err := ix.takeInWhole(ctx, ready, own, "")
|
|
if errors.Is(err, errClaimLost) {
|
|
return a, "claimed by another take-in", nil
|
|
}
|
|
if errors.Is(err, errVerdictsNotYet) {
|
|
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
|
|
}
|
|
return a, ix.waitingLine(w), err
|
|
}
|
|
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
|
|
}
|
|
|
|
// holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds
|
|
// it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry
|
|
// (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or
|
|
// indexer_drop_pending drops it. A merge unread for a transport fault (unread) is kept as Pending.Unread, not Lost, and read
|
|
// again on the next message heard on the forge (UnreadAgain). Nothing of the revision is taken in meanwhile.
|
|
func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, reasons map[string]string, unread map[string]bool) error {
|
|
t := revisionWait(takein.TakeInPending, ready, "")
|
|
for _, g := range lost {
|
|
t.Lost = append(t.Lost, g.Key)
|
|
}
|
|
said := true
|
|
for _, e := range ready {
|
|
isLost := slices.Contains(t.Lost, e.Key)
|
|
if e.Value.Said != t.Key || e.Value.SaidKind != takein.TakeInPending || (isLost && (unread[e.Key] && e.Value.Unread == "" || !unread[e.Key] && e.Value.Lost == "")) {
|
|
said = false
|
|
}
|
|
}
|
|
if said {
|
|
return nil
|
|
}
|
|
for _, g := range lost {
|
|
if unread[g.Key] {
|
|
ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge, at the start, and by %s",
|
|
g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool)
|
|
continue
|
|
}
|
|
ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; %s reads it again; indexer_drop_pending drops it once a later merge of %s covers it",
|
|
g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool, g.Forge.Repository)
|
|
}
|
|
if !ix.sayTakeIn(ctx, t) {
|
|
return nil
|
|
}
|
|
var kept []PendingEntry
|
|
for _, e := range ready {
|
|
if !slices.Contains(t.Lost, e.Key) {
|
|
kept = append(kept, e)
|
|
}
|
|
}
|
|
for _, g := range lost {
|
|
reason, isUnread := reasons[g.Key], unread[g.Key]
|
|
now := ix.now().UTC()
|
|
ix.markSaid(ctx, []PendingEntry{g}, &t, func(p *Pending) {
|
|
p.Lost, p.Unread = reason, ""
|
|
if !isUnread {
|
|
p.UnreadSince = time.Time{}
|
|
return
|
|
}
|
|
p.Lost, p.Unread = "", reason
|
|
if p.UnreadSince.IsZero() {
|
|
p.UnreadSince = now
|
|
}
|
|
})
|
|
}
|
|
ix.markSaid(ctx, kept, &t, nil)
|
|
return nil
|
|
}
|
|
|
|
// waitingLine says what waits after an evaluation, for the heard state.
|
|
func (ix *Indexer) waitingLine(w Waits) string {
|
|
var parts []string
|
|
for _, g := range slices.Sorted(maps.Keys(w.Groups)) {
|
|
parts = append(parts, fmt.Sprintf("the group %q gathered (%d held)", g, len(w.Groups[g])))
|
|
}
|
|
for _, g := range slices.Sorted(maps.Keys(w.Behind)) {
|
|
parts = append(parts, fmt.Sprintf("%d merges behind %s", len(w.Behind[g]), g))
|
|
}
|
|
return strings.Join(parts, "; ")
|
|
}
|
|
|
|
// dryRun is an indexer computing a take-in without writing it, and the overlays of what it would have written.
|
|
type dryRun struct {
|
|
ix *Indexer
|
|
modules, graph, verdicts *overlayStore
|
|
candidates, taken, predefined, configs *overlayStore
|
|
oldSections *overlayStore
|
|
}
|
|
|
|
// dry is an indexer over overlays of this one's states (overlay.go): what a take-in would write stays in memory and
|
|
// nothing reaches the bus but the manifests and the packages the revision publishes, which clause 1 writes all the
|
|
// same. It says no event, kicks nothing in the background, and knows what this one knows of the source, the toolchains
|
|
// and the previews. Every field of Indexer is named here (TestDryNamesEveryField holds it so).
|
|
func (ix *Indexer) dry() *dryRun {
|
|
over := func(st Store) (Store, *overlayStore) {
|
|
if st == nil {
|
|
return nil, nil
|
|
}
|
|
o := newOverlay(st)
|
|
return o, o
|
|
}
|
|
d := &dryRun{}
|
|
x := &Indexer{Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
|
|
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, Read: ix.Read, Contains: ix.Contains,
|
|
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil,
|
|
// The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the
|
|
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
|
|
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
|
|
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
|
|
Holds: nil, Counted: nil, AskEngines: nil, HoldBounds: nil, OwnBuildState: nil, underSeats: true}
|
|
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
|
|
x.Store, d.candidates = over(ix.Store)
|
|
x.Modules, d.modules = over(ix.Modules)
|
|
x.Graph, d.graph = over(ix.Graph)
|
|
x.Verdicts, d.verdicts = over(ix.Verdicts)
|
|
x.Taken, d.taken = over(ix.Taken)
|
|
x.Predefined, d.predefined = over(ix.Predefined)
|
|
x.ConfigCandidates, d.configs = over(ix.ConfigCandidates)
|
|
x.OldSections, d.oldSections = over(ix.OldSections)
|
|
// What the trunk holds unjudged is read through, as fresh; what the dry run would keep stays in its overlay (issue 1966).
|
|
x.Unjudged, _ = over(ix.Unjudged)
|
|
ix.mu.Lock()
|
|
x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)}
|
|
x.previews = maps.Clone(ix.previews)
|
|
x.toolchains = map[string]map[string]nodebuildgo.Toolchain{}
|
|
for l, by := range ix.toolchains {
|
|
x.toolchains[l] = maps.Clone(by)
|
|
}
|
|
x.moving = maps.Clone(ix.moving)
|
|
x.untaken = maps.Clone(ix.untaken)
|
|
ix.mu.Unlock()
|
|
ix.ownMu.Lock()
|
|
x.ownFP = ix.ownFP
|
|
ix.ownMu.Unlock()
|
|
d.ix = x
|
|
return d
|
|
}
|
|
|
|
// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees
|
|
// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "",
|
|
// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read
|
|
// is left out, as its take-in will drop it.
|
|
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) {
|
|
d := ix.dry()
|
|
// The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer
|
|
// it names is not refused for a verb or an event the built-in list lacks.
|
|
seats, err := ix.revisionSeats(ctx, ready)
|
|
if err != nil {
|
|
return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
|
|
}
|
|
var applyErr error
|
|
judge.WithSeats(seats, func() {
|
|
for _, e := range ready {
|
|
if e.Covered {
|
|
continue
|
|
}
|
|
if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil {
|
|
if errors.Is(err, ErrUnreadable) {
|
|
continue
|
|
}
|
|
applyErr = fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
|
|
return
|
|
}
|
|
}
|
|
})
|
|
if applyErr != nil {
|
|
return "", d, applyErr
|
|
}
|
|
key, _ := meshcatalogclient.ModulesKey(Module)
|
|
value, _, found, err := d.ix.Modules.Get(ctx, key)
|
|
if err != nil {
|
|
return "", d, err
|
|
}
|
|
if !found {
|
|
return NoBuild + "the indexer holds no record of itself", d, nil
|
|
}
|
|
m, err := meshcatalogclient.DecodeModule(value)
|
|
if err != nil {
|
|
return "", d, nil
|
|
}
|
|
if m.Fp2 == "" {
|
|
return NoBuild + m.None2, d, nil
|
|
}
|
|
return m.Fp2, d, nil
|
|
}
|
|
|
|
// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another
|
|
// take-in wrote the key, or the store failed) releases every claim this take-in made and answers false: nothing is
|
|
// taken in, and no claim of this one is left behind.
|
|
func (ix *Indexer) claim(ctx context.Context, entries []PendingEntry, own, by string) ([]PendingEntry, bool) {
|
|
id := fmt.Sprintf("%s at %s", own, ix.now().Format(time.RFC3339Nano))
|
|
if by != "" {
|
|
id = by + ": " + id
|
|
}
|
|
claimed := make([]PendingEntry, 0, len(entries))
|
|
for _, e := range entries {
|
|
p := e.Value
|
|
p.Taking, p.LastWriter = id, own
|
|
data, err := encodePending(p)
|
|
if err == nil {
|
|
err = ix.Pending.Update(ctx, e.Key, data, e.Revision)
|
|
}
|
|
if err != nil {
|
|
ix.logf("take-in: the claim of %s.%s is lost (%v): another take-in took it, so this one takes nothing in", PendingState, e.Key, err)
|
|
ix.release(ctx, claimed)
|
|
return nil, false
|
|
}
|
|
e.Value = p
|
|
_, rev, _, err := ix.Pending.Get(ctx, e.Key)
|
|
if err != nil {
|
|
ix.logf("take-in: the claim of %s.%s cannot be read back (%v), so this take-in takes nothing in", PendingState, e.Key, err)
|
|
ix.release(ctx, append(claimed, e))
|
|
return nil, false
|
|
}
|
|
e.Revision = rev
|
|
claimed = append(claimed, e)
|
|
}
|
|
return claimed, true
|
|
}
|
|
|
|
// release clears this take-in's claim on each entry, so a later merge, a restart or a hand take-in takes it in. It
|
|
// reads the key again and clears it by compare-and-set only while the claim is still this take-in's: a key another
|
|
// take-in wrote since, or removed, is left to it. A release that fails is said; the start of the next indexer voids
|
|
// the claim (TakeInAtStart).
|
|
func (ix *Indexer) release(ctx context.Context, claimed []PendingEntry) {
|
|
for _, e := range claimed {
|
|
value, rev, found, err := ix.Pending.Get(ctx, e.Key)
|
|
if err == nil && !found {
|
|
continue
|
|
}
|
|
var cur Pending
|
|
if err == nil {
|
|
err = json.Unmarshal(value, &cur)
|
|
}
|
|
if err == nil && cur.Taking != e.Value.Taking {
|
|
continue
|
|
}
|
|
if err == nil {
|
|
cur.Taking = ""
|
|
var data []byte
|
|
if data, err = encodePending(cur); err == nil {
|
|
err = ix.Pending.Update(ctx, e.Key, data, rev)
|
|
}
|
|
}
|
|
if err != nil {
|
|
ix.logf("take-in: the claim of %s.%s is not released (%v): the next indexer's start voids it", PendingState, e.Key, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// takeInWhole takes the entries in with this indexer, as merges are taken in (ApplyMessage), oldest first, then
|
|
// removes them from pending and says each wait they were said under taken. by names a hand act ("" for none).
|
|
func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own, by string) (Applied, error) {
|
|
return ix.takeInWholeBy(ctx, entries, own, by, "", "")
|
|
}
|
|
|
|
// takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events.
|
|
func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) {
|
|
if ix.Verdicts != nil && !ix.verdictsOnBus(ctx) && len(entries) > 0 {
|
|
// Every manifest a take-in writes is judged in it and its verdict said before take-in.taken (issue 1966 note 154,
|
|
// ADR 0668 §4): with the verdicts state not on the bus yet, nothing is taken in; its bind evaluates pending again.
|
|
ix.logf("take-in: %d merges wait, oldest %s: the state %s is not on the bus yet, so nothing is taken in until it is, and no manifest is written without its verdict",
|
|
len(entries), entries[0].Key, VerdictsBucket)
|
|
return Applied{}, errVerdictsNotYet
|
|
}
|
|
claimed, ok := ix.claim(ctx, entries, own, by)
|
|
if !ok {
|
|
return Applied{}, errClaimLost
|
|
}
|
|
var applied Applied
|
|
newest := map[string]PendingEntry{}
|
|
for _, e := range claimed {
|
|
if !e.Covered {
|
|
newest[strings.ToLower(e.Forge.Repository)] = e
|
|
}
|
|
}
|
|
for _, e := range claimed {
|
|
if e.Covered {
|
|
by := "the taken merge"
|
|
if n, ok := newest[strings.ToLower(e.Forge.Repository)]; ok {
|
|
by = n.Key + ", the newest merge"
|
|
}
|
|
ix.logf("take-in: %s (%s#%d) is covered by %s of %s in this revision: its message is not read, and it is ended with it", e.Key, e.Forge.Repository, e.Forge.Number, by, e.Forge.Repository)
|
|
// Its pull request leaves its group's candidate as a merged message takes it out; nothing else of it is applied.
|
|
if err := ix.removeCandidate(ctx, e.Forge); err != nil {
|
|
ix.release(ctx, claimed)
|
|
return applied, fmt.Errorf("taking %s (%s#%d) out of its candidate: %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
|
|
}
|
|
continue
|
|
}
|
|
a, err := ix.ApplyMessage(ctx, e.Forge)
|
|
applied.Written = append(applied.Written, a.Written...)
|
|
if a.Late != "" {
|
|
applied.Late = a.Late
|
|
}
|
|
if errors.Is(err, ErrUnreadable) {
|
|
ix.logf("take-in: %s (%s#%d) can never be read, dropped: %v", e.Key, e.Forge.Repository, e.Forge.Number, err)
|
|
continue
|
|
}
|
|
if err != nil {
|
|
ix.release(ctx, claimed)
|
|
return applied, fmt.Errorf("taking %s (%s#%d) in: %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
|
|
}
|
|
}
|
|
for _, e := range claimed {
|
|
if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil {
|
|
ix.logf("take-in: %s.%s is taken in and not removed: %v", PendingState, e.Key, err)
|
|
}
|
|
}
|
|
ix.forget(claimed...)
|
|
ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s", len(claimed), own, byWords(by), claimed[0].Key)
|
|
for _, t := range takenWaits(claimed) {
|
|
t.By, t.Why = who, why
|
|
ix.sayTakeIn(ctx, t)
|
|
}
|
|
return applied, nil
|
|
}
|
|
|
|
// takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait
|
|
// indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and
|
|
// the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait
|
|
// the watchman learned of without the indexer marking it (seeded from indexer_pending, or said by the own-build-unknown
|
|
// path) ends too (design 47, Taken in by the indexer it names). No key is said twice.
|
|
func takenWaits(entries []PendingEntry) []takein.TakeIn {
|
|
out := saidWaits(entries)
|
|
said := map[string]bool{}
|
|
for _, t := range out {
|
|
said[t.Key] = true
|
|
}
|
|
w := WaitsOf(entries)
|
|
var named []takein.TakeIn
|
|
if len(w.Ready) > 0 {
|
|
named = append(named, revisionWait(takein.TakeInTaken, w.Ready, w.Ready[len(w.Ready)-1].Value.WaitsFor))
|
|
}
|
|
for _, g := range slices.Sorted(maps.Keys(w.Groups)) {
|
|
named = append(named, groupWait(takein.TakeInTaken, g, w.Groups[g]))
|
|
}
|
|
for _, g := range slices.Sorted(maps.Keys(w.Behind)) {
|
|
named = append(named, revisionWait(takein.TakeInTaken, w.Behind[g], w.Behind[g][len(w.Behind[g])-1].Value.WaitsFor))
|
|
}
|
|
for _, e := range entries {
|
|
named = append(named, revisionWait(takein.TakeInTaken, []PendingEntry{e}, e.Value.WaitsFor))
|
|
}
|
|
for _, t := range named {
|
|
if !said[t.Key] {
|
|
said[t.Key] = true
|
|
out = append(out, t)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func byWords(by string) string {
|
|
if by == "" {
|
|
return ""
|
|
}
|
|
return " (a hand act: " + by + ")"
|
|
}
|
|
|
|
// saidWaits are the taken events of every wait the entries were said under that is not ended yet.
|
|
func saidWaits(entries []PendingEntry) []takein.TakeIn {
|
|
by := map[string][]PendingEntry{}
|
|
for _, e := range entries {
|
|
if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending {
|
|
by[e.Value.Said] = append(by[e.Value.Said], e)
|
|
}
|
|
}
|
|
var out []takein.TakeIn
|
|
for _, key := range slices.Sorted(maps.Keys(by)) {
|
|
held := by[key]
|
|
if strings.HasPrefix(key, "g.") {
|
|
out = append(out, groupWait(takein.TakeInTaken, held[0].Value.Group, held))
|
|
continue
|
|
}
|
|
t := revisionWait(takein.TakeInTaken, held, held[0].Value.WaitsFor)
|
|
t.Key = key
|
|
out = append(out, t)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving
|
|
// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its
|
|
// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said
|
|
// pending under its oldest merge's key, and a wait it was said under before by another key is said taken.
|
|
func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error {
|
|
key, _ := meshcatalogclient.ModulesKey(Module)
|
|
if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil {
|
|
return fmt.Errorf("the indexer's own module entry: %w", err)
|
|
}
|
|
if gk, err := graph.Key(Module); err == nil {
|
|
if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) {
|
|
return fmt.Errorf("the indexer's key of the module graph: %w", err)
|
|
}
|
|
}
|
|
// Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the
|
|
// revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost).
|
|
refused := ""
|
|
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
|
|
if m, err := meshcatalogclient.DecodeModule(value); err == nil && m.Manifest != "" {
|
|
if vk, err := meshcatalogclient.VerdictsKey(m.Manifest); err == nil {
|
|
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
|
|
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
|
|
}
|
|
if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok {
|
|
if v, err := meshcatalogclient.DecodeVerdict(vv); err == nil && meshcatalogclient.VerdictRefuses(v) {
|
|
refused = strings.Join(v.Problems, "; ")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
t := revisionWait(takein.TakeInPending, ready, names)
|
|
t.Refused = refused
|
|
var ended, unmarked []PendingEntry
|
|
said, changed := false, false
|
|
for _, e := range ready {
|
|
switch {
|
|
case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending:
|
|
said = true
|
|
changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused
|
|
if e.Value.WaitsFor != names || e.Value.Refused != refused {
|
|
unmarked = append(unmarked, e)
|
|
}
|
|
case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending:
|
|
ended = append(ended, e)
|
|
unmarked = append(unmarked, e)
|
|
default:
|
|
unmarked = append(unmarked, e)
|
|
}
|
|
}
|
|
for _, old := range saidWaits(ended) {
|
|
ix.sayTakeIn(ctx, old)
|
|
}
|
|
if said && !changed {
|
|
// Nothing to say on the bus, but the start decided nothing and a reader of the journal sees why: the build learnt, the build the wait names.
|
|
ix.logf("take-in: %d merges, oldest %s, still name the indexer build %s, this program runs %s: they wait, as already said", len(ready), t.Key, names, own)
|
|
}
|
|
if !said || changed {
|
|
ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own)
|
|
if refused != "" {
|
|
ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused)
|
|
}
|
|
if !ix.sayTakeIn(ctx, t) {
|
|
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
|
|
return nil
|
|
}
|
|
unmarked = ready
|
|
}
|
|
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
|
|
return nil
|
|
}
|
|
|
|
// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it.
|
|
func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error {
|
|
if from == nil || to == nil {
|
|
return nil
|
|
}
|
|
value, deleted, ok := from.written(key)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return write(ctx, to, key, func(cur []byte, found bool) ([]byte, bool, error) {
|
|
switch {
|
|
case deleted:
|
|
return nil, found, nil
|
|
case found && string(cur) == string(value):
|
|
return nil, false, nil
|
|
}
|
|
return value, false, nil
|
|
})
|
|
}
|
|
|
|
// PendingWaits is what indexer_pending answers of the pending state's entries (ADR 0633 §5): the ready merges as one
|
|
// revision, each group being gathered, and the merges behind a group, each with the modules it holds back and the
|
|
// last take-in event said of it.
|
|
func PendingWaits(entries []PendingEntry) []takein.PendingWait {
|
|
w := WaitsOf(entries)
|
|
var out []takein.PendingWait
|
|
if len(w.Ready) > 0 {
|
|
out = append(out, waitOf(w.Ready, takein.TakeInRevision, ""))
|
|
}
|
|
for _, g := range slices.Sorted(maps.Keys(w.Groups)) {
|
|
out = append(out, waitOf(w.Groups[g], takein.TakeInGroup, ""))
|
|
}
|
|
for _, g := range slices.Sorted(maps.Keys(w.Behind)) {
|
|
out = append(out, waitOf(w.Behind[g], takein.TakeInRevision, g))
|
|
}
|
|
slices.SortStableFunc(out, func(a, b takein.PendingWait) int { return a.Since.Compare(b.Since) })
|
|
return out
|
|
}
|
|
|
|
func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
|
|
pw := takein.PendingWait{Key: entries[0].Key, What: what, Merges: mergesOf(entries), Behind: behind, Since: sinceOf(entries),
|
|
WaitsFor: entries[len(entries)-1].Value.WaitsFor, Holds: []string{}}
|
|
if what == takein.TakeInGroup {
|
|
pw.Group = entries[0].Value.Group
|
|
pw.Key = takein.GroupTakeInKey(pw.Group)
|
|
pw.Members = entries[len(entries)-1].Forge.Members
|
|
pw.WaitsFor = ""
|
|
}
|
|
for _, e := range entries {
|
|
pw.Keys = append(pw.Keys, e.Key)
|
|
for _, name := range e.Holds {
|
|
if !slices.Contains(pw.Holds, name) {
|
|
pw.Holds = append(pw.Holds, name)
|
|
}
|
|
}
|
|
if e.Value.Lost != "" {
|
|
pw.Lost = append(pw.Lost, e.Key)
|
|
}
|
|
if e.Value.Unread != "" {
|
|
pw.Unread = append(pw.Unread, e.Key)
|
|
}
|
|
if e.Value.Refused != "" {
|
|
pw.Refused = e.Value.Refused
|
|
}
|
|
if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) {
|
|
pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt
|
|
}
|
|
}
|
|
slices.Sort(pw.Holds)
|
|
return pw
|
|
}
|
|
|
|
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the
|
|
// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge,
|
|
// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are
|
|
// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after.
|
|
func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn {
|
|
r := TakenIn{Module: key}
|
|
if ix.Pending == nil {
|
|
r.Answer = "the pending state is not on the bus: nothing waits"
|
|
return r
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
r.Answer = err.Error()
|
|
return r
|
|
}
|
|
w := WaitsOf(entries)
|
|
var take []PendingEntry
|
|
switch {
|
|
case strings.HasPrefix(key, "g."):
|
|
for g, held := range w.Groups {
|
|
if takein.GroupTakeInKey(g) == key {
|
|
take = held
|
|
}
|
|
}
|
|
case len(w.Ready) > 0 && w.Ready[0].Key == key:
|
|
take = w.Ready
|
|
}
|
|
if len(take) == 0 {
|
|
r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key)
|
|
return r
|
|
}
|
|
take, lost, reasons, _ := ix.loadNewest(ctx, take)
|
|
if len(lost) > 0 {
|
|
r.Answer = fmt.Sprintf("nothing is taken in: the message of %s is not read (%s); asked again, it is read again; indexer_drop_pending drops it once a later merge of its repository covers it", lost[0].Key, reasons[lost[0].Key])
|
|
return r
|
|
}
|
|
ix.logf("take-in: a hand take-in of %s (%d merges) asked by %s: %s", key, len(take), by, why)
|
|
hand := fmt.Sprintf("%s (%s)", by, why)
|
|
if _, err := ix.takeInWholeBy(ctx, take, own, hand, by, why); err != nil {
|
|
r.Answer = err.Error()
|
|
return r
|
|
}
|
|
r.Stored = true
|
|
r.Answer = fmt.Sprintf("%d merges taken in by the running indexer %s", len(take), own)
|
|
if _, _, err := ix.evaluate(ctx, own, false); err != nil {
|
|
r.Answer += "; what waited behind it is not evaluated: " + err.Error()
|
|
}
|
|
return r
|
|
}
|
|
|
|
// DropPending is the hand drop (indexer_drop_pending, ADR 0633 §5): the one pending merge under key, abandoned
|
|
// because a later merge replaced it, is removed and nothing of it is taken in. Its wait is said taken, naming the drop;
|
|
// the merges left under that wait are said pending again by the evaluation that follows. A merge that no later pending
|
|
// merge and no taken merge of its repository covers is refused unless abandon (issue 1966: #650, its repository's head,
|
|
// was dropped on the advice of a line that named a drop where nothing covered it; ADR 0668 §5).
|
|
func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon bool) TakenIn {
|
|
r := TakenIn{Module: key}
|
|
if ix.Pending == nil {
|
|
r.Answer = "the pending state is not on the bus: nothing waits"
|
|
return r
|
|
}
|
|
own := ix.ownBuildNow(ctx)
|
|
ix.pendingMu.Lock()
|
|
defer ix.pendingMu.Unlock()
|
|
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
|
if err != nil {
|
|
r.Answer = err.Error()
|
|
return r
|
|
}
|
|
at := slices.IndexFunc(entries, func(e PendingEntry) bool { return e.Key == key })
|
|
if at < 0 {
|
|
r.Answer = fmt.Sprintf("%s holds no merge under %s", PendingState, key)
|
|
return r
|
|
}
|
|
e := entries[at]
|
|
if !abandon {
|
|
// Covered: a later merge of the same repository, pending or taken in, has this merge's commit as an ancestor (ADR
|
|
// 0668 §6), asked of the forge (older); a later merge whose history does not hold it covers nothing.
|
|
covered := false
|
|
for _, o := range entries[at+1:] {
|
|
if o.Forge.Repository != e.Forge.Repository {
|
|
continue
|
|
}
|
|
in, err := ix.older(ctx, e.Forge.Repository, e.Forge.Merge, o.Forge.Merge)
|
|
if err != nil {
|
|
r.Answer = fmt.Sprintf("%s is not dropped: whether the later pending merge %s covers it is not read: %v", key, o.Key, err)
|
|
return r
|
|
}
|
|
if in {
|
|
covered = true
|
|
break
|
|
}
|
|
}
|
|
if !covered {
|
|
taken, err := ix.takenAlready(ctx, forgemsg.GroupMerge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge})
|
|
if err != nil {
|
|
r.Answer = fmt.Sprintf("%s is not dropped: whether a taken merge of %s covers it is not read: %v", key, e.Forge.Repository, err)
|
|
return r
|
|
}
|
|
covered = taken
|
|
}
|
|
if !covered {
|
|
r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s, pending or taken in, has %s as an ancestor, so its "+
|
|
"changes would never be taken in (ADR 0633 §5); %s reads its message again, or abandon: true drops it all the same",
|
|
key, e.Forge.Repository, e.Forge.Merge, TakeInPendingTool)
|
|
return r
|
|
}
|
|
}
|
|
if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil {
|
|
r.Answer = fmt.Sprintf("%s.%s is not removed: %v", PendingState, key, err)
|
|
return r
|
|
}
|
|
ix.forget(e)
|
|
ix.logf("take-in: %s (%s#%d at %s) dropped by hand, asked by %s: %s; nothing of it is taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge, by, why)
|
|
if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending {
|
|
var same []PendingEntry
|
|
for _, o := range entries {
|
|
if o.Key != key && o.Value.Said == e.Value.Said {
|
|
same = append(same, o)
|
|
}
|
|
}
|
|
for _, t := range saidWaits([]PendingEntry{e}) {
|
|
t.By, t.Why, t.Dropped = by, why, true
|
|
ix.sayTakeIn(ctx, t)
|
|
}
|
|
ix.markSaid(ctx, same, nil, func(p *Pending) { p.Said, p.SaidKind = "", "" })
|
|
}
|
|
r.Stored, r.Answer = true, fmt.Sprintf("%s dropped: %s#%d at %s is not taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge)
|
|
if own != "" {
|
|
if _, _, err := ix.evaluate(ctx, own, false); err != nil {
|
|
r.Answer += "; what waits is not evaluated again: " + err.Error()
|
|
}
|
|
}
|
|
return r
|
|
}
|
|
|
|
// TakeInMerge is the hand take-in of one merge (indexer_take_in_merge, issue 1585): its merged message is asked of the forge
|
|
// bridge and handed to the take-in as a delivered merged message is (Arrive, under the stream sequence seq, which keeps the
|
|
// merge order), so it is held for the indexer its revision names or taken in, and a merge already held under that
|
|
// sequence is not written twice. For a merge the take-in never took in. The bridge's own words answer when it cannot give
|
|
// the message.
|
|
func (ix *Indexer) TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) TakenIn {
|
|
r := TakenIn{Module: takein.PendingKey(seq)}
|
|
if ix.Message == nil {
|
|
r.Answer = "this indexer does not ask the forge bridge"
|
|
return r
|
|
}
|
|
data, err := ix.Message(ctx, repository, merge, 0)
|
|
if err != nil {
|
|
r.Answer = fmt.Sprintf("the forge bridge could not answer for the merge of %s at %s: %v", repository, merge, err)
|
|
return r
|
|
}
|
|
f, err := forgemsg.DecodeForge(data)
|
|
if err != nil {
|
|
r.Answer = fmt.Sprintf("the forge bridge's answer for %s at %s is not a forge message: %v", repository, merge, err)
|
|
return r
|
|
}
|
|
if f.Kind != forgemsg.ForgeMerged || f.Merge != merge || f.Repository != repository {
|
|
r.Answer = fmt.Sprintf("the forge bridge's answer is not the merge of %s at %s", repository, merge)
|
|
return r
|
|
}
|
|
ix.logf("take-in: a hand take-in of the merge %s#%d at %s (sequence %d) asked by %s: %s", f.Repository, f.Number, f.Merge, seq, by, why)
|
|
a, words, err := ix.Arrive(ctx, seq, f)
|
|
if err != nil {
|
|
r.Answer = err.Error()
|
|
return r
|
|
}
|
|
r.Stored, r.Repository, r.Commit = true, f.Repository, f.Merge
|
|
r.Answer = fmt.Sprintf("%s#%d at %s handed to the take-in as a delivered merge (%d keys written)", f.Repository, f.Number, f.Merge, len(a.Written))
|
|
if words != "" {
|
|
r.Answer += "; " + words
|
|
}
|
|
return r
|
|
}
|