Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/pending.go
T

1168 lines
46 KiB
Go

package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
// The take-in step (novox/hq ADR 0633): a source revision is taken in only by the indexer it names. A merged message is
// kept in the indexer's own state pending, one key per merge, until the take-in that ends it; a revision naming another
// build of the indexer writes only what building and moving that indexer needs, and the rest stays pending for it.
const (
// PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed
// takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR
// 0595 §11), history 1, reached by the indexer alone.
PendingState = "pending"
PendingBucket = delivery.IndexerModule + "_" + PendingState
// PendingTool reads it (ADR 0633 §5); TakeInPendingTool and DropPendingTool steer it.
PendingTool = "indexer_pending"
TakeInPendingTool = "indexer_take_in_pending"
DropPendingTool = "indexer_drop_pending"
)
// errClaimLost says another take-in claimed a pending merge first: this one took nothing in.
var errClaimLost = errors.New("another take-in claimed the merges first, so this one took nothing in")
// NoBuild opens what a revision waits for when it gives the indexer no fingerprint: the reason follows. No indexer
// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending).
const NoBuild = "no build: "
// ErrOwnBuildUnknown is what a merged message answers while the indexer cannot learn the build it runs (ADR 0633 §1):
// the message is left unacknowledged, so the bus delivers it again, and nothing is written.
var ErrOwnBuildUnknown = errors.New("the build of the indexer this program runs is not known")
// PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many
// modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out
// from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it.
type PendingRef struct {
Repository string `json:"repository"`
Number int `json:"number,omitempty"`
Merge string `json:"merge"`
// Branch is the pull request's branch, which names its candidate: a merge covered by a newer one is taken out of it.
Branch string `json:"branch,omitempty"`
Members []delivery.Head `json:"members,omitempty"`
// Holds are the modules the merge's manifests name, which indexer_pending says it holds back.
Holds []string `json:"holds,omitempty"`
}
// Pending is one value of the pending state: a reference to the merged message (Seq, its sequence on the forge bridge's
// stream, and Ref), or, in a value an earlier indexer wrote, the message whole as the forge bridge encoded it; the group it
// was merged in (Ended once the group's end was heard), since when it is held, the indexer build its revision waits
// for (absent while it waits for nothing yet, or behind a group being gathered), the build of the indexer that last
// wrote it, the wait it was last said under on the take-in stream (Said, SaidAt), and Taking, set by a take-in that
// claimed it: a second indexer that finds it claimed takes nothing in (ADR 0633 §1: never two take-ins of one merge).
type Pending struct {
V int `json:"v"`
Seq uint64 `json:"seq,omitempty"`
Ref *PendingRef `json:"ref,omitempty"`
Forge json.RawMessage `json:"forge,omitempty"`
Lost string `json:"lost,omitempty"`
Group string `json:"group,omitempty"`
Ended bool `json:"ended,omitempty"`
Since time.Time `json:"since"`
WaitsFor string `json:"waits-for,omitempty"`
LastWriter string `json:"last-writer"`
Said string `json:"said,omitempty"`
SaidKind string `json:"said-kind,omitempty"`
SaidAt time.Time `json:"said-at,omitzero"`
Taking string `json:"taking,omitempty"`
}
// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a
// value that keeps a reference, Forge holds only the fields of the reference (Light) until Indexer.load reads the message
// from the stream; Holds are the modules the merge's manifests name.
type PendingEntry struct {
Key string
Revision uint64
Value Pending
Forge forgemsg.Forge
Light bool
Holds []string
// Covered says a newer merge of the same repository is in the same revision: its message is not read or applied, since
// applying the newest writes every module whose manifest differs from what is held (modules.go).
Covered bool
}
// DecodePending reads one pending value and its merged message.
func DecodePending(data []byte) (Pending, forgemsg.Forge, error) {
var p Pending
if err := json.Unmarshal(data, &p); err != nil {
return p, forgemsg.Forge{}, fmt.Errorf("not a pending value: %w", err)
}
if p.V != 1 {
return p, forgemsg.Forge{}, fmt.Errorf("a pending value of version %d, not 1", p.V)
}
if len(p.Forge) == 0 && p.Ref != nil {
r := p.Ref
return p, forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: r.Repository, Number: r.Number,
Merge: r.Merge, Branch: r.Branch, Group: p.Group, Members: r.Members}, nil
}
f, err := forgemsg.DecodeForge(p.Forge)
if err != nil {
return p, f, fmt.Errorf("its merged message: %w", err)
}
if f.Kind != forgemsg.ForgeMerged {
return p, f, fmt.Errorf("a pending value holds a %s message, not a merged one", f.Kind)
}
return p, f, nil
}
func encodePending(p Pending) ([]byte, error) {
p.V = 1
return json.Marshal(p)
}
// ReadPending is every value the pending state holds, in key order (the merge order); a key that does not read is
// said through logf and left.
func ReadPending(ctx context.Context, st Store, logf func(string, ...any)) ([]PendingEntry, error) {
keys, err := st.Keys(ctx)
if err != nil {
return nil, fmt.Errorf("listing %s: %w", PendingState, err)
}
slices.Sort(keys)
var out []PendingEntry
for _, key := range keys {
value, rev, found, err := st.Get(ctx, key)
if err != nil {
return nil, fmt.Errorf("%s.%s: %w", PendingState, key, err)
}
if !found {
continue
}
p, f, err := DecodePending(value)
if err != nil {
if logf != nil {
logf("%s.%s is left: %v", PendingState, key, err)
}
continue
}
e := PendingEntry{Key: key, Revision: rev, Value: p, Forge: f, Light: len(p.Forge) == 0}
if p.Ref != nil {
e.Holds = p.Ref.Holds
} else {
e.Holds = holdsOf(f)
}
out = append(out, e)
}
return out, nil
}
// holdsOf are the modules a merged message's manifests name, sorted.
func holdsOf(f forgemsg.Forge) []string {
var out []string
for _, raw := range f.Manifests {
if name := manifestModule(raw); name != "" && !slices.Contains(out, name) {
out = append(out, name)
}
}
slices.Sort(out)
return out
}
// refOf is what a pending value keeps of the merged message f (PendingRef).
func refOf(f forgemsg.Forge) *PendingRef {
return &PendingRef{Repository: f.Repository, Number: f.Number, Merge: f.Merge, Branch: f.Branch, Members: f.Members, Holds: holdsOf(f)}
}
// heldMessage is the message Arrive was handed for a sequence, kept in memory until its merge is taken in or dropped.
func (ix *Indexer) heldMessage(seq uint64) (forgemsg.Forge, bool) {
ix.heldMu.Lock()
defer ix.heldMu.Unlock()
f, ok := ix.held[seq]
return f, ok
}
func (ix *Indexer) hold(seq uint64, f forgemsg.Forge) {
ix.heldMu.Lock()
defer ix.heldMu.Unlock()
if ix.held == nil {
ix.held = map[uint64]forgemsg.Forge{}
}
ix.held[seq] = f
}
func (ix *Indexer) forget(entries ...PendingEntry) {
ix.heldMu.Lock()
defer ix.heldMu.Unlock()
for _, e := range entries {
delete(ix.held, e.Value.Seq)
}
}
// load gives the entry the merged message whole: the copy an earlier value holds, the message Arrive was handed (kept in
// memory), else the message read from the forge bridge's stream by the value's sequence. A stream that no longer holds it
// answers ErrForgeGone, a bus that refuses the read its error; a message that is not the merge the value refers to (a
// stream rebuilt under the same sequence) is refused.
func (ix *Indexer) load(ctx context.Context, e PendingEntry) (PendingEntry, error) {
if !e.Light {
return e, nil
}
if f, ok := ix.heldMessage(e.Value.Seq); ok {
e.Forge, e.Light = f, false
return e, nil
}
if ix.Message == nil {
return e, fmt.Errorf("the forge bridge is not asked by this indexer, so %s (%s#%d) cannot be read", e.Key, e.Forge.Repository, e.Forge.Number)
}
data, err := ix.Message(ctx, e.Forge.Repository, e.Forge.Merge)
if err != nil {
return e, fmt.Errorf("the forge bridge could not answer for the merge of %s#%d at %s: %w", e.Forge.Repository, e.Forge.Number, e.Forge.Merge, err)
}
f, err := forgemsg.DecodeForge(data)
if err != nil {
return e, fmt.Errorf("the forge bridge's answer for %s is not a forge message: %w", e.Key, err)
}
if f.Kind != forgemsg.ForgeMerged || f.Repository != e.Forge.Repository || f.Merge != e.Forge.Merge {
return e, fmt.Errorf("the forge bridge's answer is not the merge %s#%d at %s that %s refers to", e.Forge.Repository, e.Forge.Number, e.Forge.Merge, e.Key)
}
e.Forge, e.Light = f, false
return e, nil
}
// loadNewest prepares the ready merges of one revision, in merge order. Of the merges of one repository only the newest is
// loaded: applying it writes every module whose manifest differs from what is held, so it covers the older ones, which are
// marked Covered and never read. A newest merge the forge bridge cannot give is answered in lost with its reason, by key, and its revision is held (holdLost).
func (ix *Indexer) loadNewest(ctx context.Context, entries []PendingEntry) (out, lost []PendingEntry, reasons map[string]string) {
out = slices.Clone(entries)
reasons = map[string]string{}
newest := map[string]bool{}
for i := len(out) - 1; i >= 0; i-- {
repo := strings.ToLower(out[i].Forge.Repository)
if newest[repo] {
out[i].Covered = true
continue
}
newest[repo] = true
full, err := ix.load(ctx, out[i])
if err != nil {
lost = append(lost, out[i])
reasons[out[i].Key] = err.Error()
continue
}
out[i] = full
}
slices.Reverse(lost)
return out, lost, reasons
}
// Waits is the pending state read as what waits (ADR 0633 §3, §4): Ready are the merges no group still being gathered
// holds back, in merge order, which are one revision together; Groups are the groups still being gathered, each with
// its members held so far; Behind are the merges in a gathered group's member repositories that wait behind it, by the
// group's take-in key.
type Waits struct {
Ready []PendingEntry
Groups map[string][]PendingEntry
Behind map[string][]PendingEntry
}
// WaitsOf works out what waits from the entries, in key order. A member of a group whose end was not heard is
// gathered; from the first such member on, a later merge in one of the group's members' repositories waits behind it,
// because its tree holds that member's commit. Every other merge is ready.
func WaitsOf(entries []PendingEntry) Waits {
w := Waits{Groups: map[string][]PendingEntry{}, Behind: map[string][]PendingEntry{}}
blocked := map[string]string{} // repository, lower case, to the group key it waits behind
for _, e := range entries {
if e.Value.Group != "" && !e.Value.Ended {
w.Groups[e.Value.Group] = append(w.Groups[e.Value.Group], e)
for _, m := range e.Forge.Members {
if _, ok := blocked[strings.ToLower(m.Repository)]; !ok {
blocked[strings.ToLower(m.Repository)] = takein.GroupTakeInKey(e.Value.Group)
}
}
continue
}
if g, ok := blocked[strings.ToLower(e.Forge.Repository)]; ok {
w.Behind[g] = append(w.Behind[g], e)
continue
}
w.Ready = append(w.Ready, e)
}
return w
}
// mergesOf are the merges of entries, oldest first.
func mergesOf(entries []PendingEntry) []takein.Merge {
out := make([]takein.Merge, len(entries))
for i, e := range entries {
out[i] = takein.Merge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge}
}
return out
}
func sinceOf(entries []PendingEntry) time.Time {
var at time.Time
for _, e := range entries {
if at.IsZero() || e.Value.Since.Before(at) {
at = e.Value.Since
}
}
return at
}
// groupWait is the take-in body of a group being gathered.
func groupWait(kind, group string, held []PendingEntry) takein.TakeIn {
t := takein.TakeIn{Kind: kind, Key: takein.GroupTakeInKey(group), What: takein.TakeInGroup, Group: group,
Merges: mergesOf(held), Since: sinceOf(held)}
if len(held) > 0 {
t.Members = held[len(held)-1].Forge.Members
}
return t
}
// revisionWait is the take-in body of a revision: its key is its oldest merge's.
func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.TakeIn {
return takein.TakeIn{Kind: kind, Key: ready[0].Key, What: takein.TakeInRevision, Merges: mergesOf(ready),
WaitsFor: waitsFor, Since: sinceOf(ready)}
}
// pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears
// (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read
// that fails otherwise counts as on the bus: the step that follows answers the error.
func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
_, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0))
return !errors.Is(err, process.ErrNotYet)
}
// ownBuildNow is the build this program runs: the one learnt, else asked of the node-engine now.
func (ix *Indexer) ownBuildNow(ctx context.Context) string {
ix.ownMu.Lock()
fp := ix.ownFP
ix.ownMu.Unlock()
if fp != "" || ix.OwnBuild == nil {
return fp
}
ix.learnOwnBuild(ctx)
ix.ownMu.Lock()
defer ix.ownMu.Unlock()
return ix.ownFP
}
// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of
// each wait from the state. A publish that fails is said in the log: the next evaluation says it again.
func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool {
if ix.SayTakeIn == nil {
return true
}
if err := ix.SayTakeIn(ctx, t); err != nil {
ix.logf("take-in: %s.%s is not said: %v", t.Kind, t.Key, err)
return false
}
return true
}
// markSaid writes on each entry the wait it was said under, by compare-and-set; a lost one is left (its next
// evaluation says it again).
func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *takein.TakeIn, change func(*Pending)) {
for _, e := range entries {
p := e.Value
if t != nil {
p.Said, p.SaidKind, p.SaidAt = t.Key, t.Kind, ix.now()
}
if change != nil {
change(&p)
}
data, err := encodePending(p)
if err != nil {
continue
}
if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil {
ix.logf("take-in: %s.%s is not written again: %v", PendingState, e.Key, err)
}
}
}
// Arrive takes one merged message of the forge bridge's stream in under ADR 0633: seq is its stream sequence. It is
// written to pending first (a redelivered message finds its key there and writes no second), and then what pending
// holds is evaluated (Evaluate): taken in whole when its revision names the build this program runs, or held for the
// indexer it names. It answers what was taken in, and a line saying what waits. A message is never written while the
// build this program runs is not known (ErrOwnBuildUnknown).
func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Applied, string, error) {
if ix.Pending == nil || ix.Modules == nil || !ix.pendingOnBus(ctx) {
// The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before,
// whether or not the build this program runs is known.
a, err := ix.ApplyMessage(ctx, f)
return a, "", err
}
own := ix.ownBuildNow(ctx)
if own == "" {
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
key := takein.PendingKey(seq)
ix.hold(seq, f)
data, err := encodePending(Pending{Seq: seq, Ref: refOf(f), Group: f.Group, Since: ix.now(), LastWriter: own})
if err != nil {
return Applied{}, "", err
}
switch err := ix.Pending.Create(ctx, key, data); {
case errors.Is(err, process.ErrNotYet):
// The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before.
ix.logf("take-in: the state %s is not on the bus yet, so %s#%d is taken in at once, as before ADR 0633", PendingBucket, f.Repository, f.Number)
a, err := ix.ApplyMessage(ctx, f)
return a, "", err
case err != nil && !errors.Is(err, ErrConflict):
return Applied{}, "", fmt.Errorf("%s.%s: %w", PendingState, key, err)
}
if f.Group != "" {
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
return Applied{}, "", err
}
if held := WaitsOf(entries).Groups[f.Group]; len(held) > 0 {
t := groupWait(takein.TakeInPending, f.Group, held)
if ix.sayTakeIn(ctx, t) {
ix.markSaid(ctx, held, &t, nil)
}
}
}
return ix.evaluate(ctx, own, false)
}
// EndGroup takes a group's end in (ADR 0633 §4): once every member it names is held, the group's members are marked
// ended, the group's wait is said taken (take-in.taken under its group key), and pending is evaluated, the group now
// one revision. A member named and neither held nor taken in yet answers an error, so the end is delivered again once
// the member arrived. An end of a group taken in already (every member's merge at or behind its repository's taken
// merge) changes nothing and is said: it is never delivered again for ever.
func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Applied, string, error) {
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
return Applied{}, "no pending state: the group's merges were taken in one by one", nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if errors.Is(err, process.ErrNotYet) {
return Applied{}, "the pending state is not on the bus yet: the group's merges were taken in one by one", nil
}
if err != nil {
return Applied{}, "", err
}
var held []PendingEntry
for _, e := range entries {
if e.Value.Group == end.Group {
held = append(held, e)
}
}
isHeld := func(m forgemsg.GroupMerge) bool {
return slices.ContainsFunc(held, func(e PendingEntry) bool {
return delivery.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number
})
}
var missing []string
for _, m := range end.Members {
if isHeld(m) {
continue
}
taken, err := ix.takenAlready(ctx, m)
if err != nil {
return Applied{}, "", err
}
if !taken {
missing = append(missing, fmt.Sprintf("%s#%d", m.Repository, m.Number))
}
}
if len(missing) > 0 {
return Applied{}, "", fmt.Errorf("the end of the group %q names %s, not held yet: it waits for their merged messages", end.Group, strings.Join(missing, ", "))
}
if len(held) == 0 {
ix.logf("take-in: the end of the group %q comes after its take-in: every member is taken in already, nothing changes", end.Group)
return Applied{}, "the group is taken in already", nil
}
if !slices.ContainsFunc(held, func(e PendingEntry) bool { return !e.Value.Ended }) {
return ix.evaluate(ctx, own, false)
}
// The group's wait ends first (take-in.taken under its group key); a revision it then is, pending for another
// indexer, is said pending under its oldest merge's key by the evaluation that follows.
t := groupWait(takein.TakeInTaken, end.Group, held)
said := &t
if !ix.sayTakeIn(ctx, t) {
said = nil // said again by the take-in or the hold that follows (saidWaits)
}
ix.markSaid(ctx, held, said, func(p *Pending) { p.Ended = true })
ix.logf("take-in: the group %q ended with %d members held: one revision now", end.Group, len(held))
return ix.evaluate(ctx, own, false)
}
// takenAlready says whether a group member's merge is at or behind its repository's taken merge.
func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (bool, error) {
if ix.Taken == nil {
return false, nil
}
key, err := delivery.RepositoryKey(m.Repository)
if err != nil {
return false, err
}
value, _, found, err := ix.Taken.Get(ctx, key)
if errors.Is(err, process.ErrNotYet) || (err == nil && !found) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("taken.%s: %w", key, err)
}
cur, err := delivery.DecodeTaken(value)
if err != nil || cur.Merge == "" {
return false, nil
}
if cur.Merge == m.Merge {
return true, nil
}
return ix.older(ctx, m.Repository, m.Merge, cur.Merge)
}
// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts:
// it evaluates pending once with the build this program runs, taking in the whole revisions that name this build; a
// group still being gathered stays, and a revision naming yet another build is held for that one. A claim left by an
// indexer that ran before is void here. When the build is not known (the node-engine did not answer), it waits for
// nothing: it says so and the consumer starts all the same; a merged message is then left unacknowledged until the
// build is known (clause 1), and its redelivery, or a toolchain report (ToolchainReported), asks again. No timer.
func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) {
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
// Nothing can wait in a state not on the bus: the start asks no build and does not wait for one.
return Applied{}, "the pending state is not on the bus yet: nothing waits", nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
ix.logf("take-in: the build this program runs is not known at its start, so nothing pending is taken in now; the forge consumer starts, and a merged message left unacknowledged asks it again")
return Applied{}, "the build this program runs is not known yet: nothing pending is taken in at the start", nil
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
a, words, err := ix.evaluate(ctx, own, true)
if errors.Is(err, process.ErrNotYet) {
return a, "the pending state is not on the bus yet: nothing waits", nil
}
return a, words, err
}
// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report
// (ADR 0633 §1, §3): a revision that now names the running build is taken in, and one that names another build is held
// for that one. It does nothing while the pending state is not on the bus or the build this program runs is not known.
func (ix *Indexer) PendingAgain(ctx context.Context) error {
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
return nil
}
own := ix.ownBuildNow(ctx)
if own == "" {
return nil
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
_, words, err := ix.evaluate(ctx, own, false)
if errors.Is(err, process.ErrNotYet) {
return nil
}
if err == nil && words != "" {
ix.logf("take-in: evaluated again after a toolchain report: %s", words)
}
return err
}
// evaluate works out what pending's ready merges name, together (ADR 0633 §3), and takes them in or holds them.
func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applied, string, error) {
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
return Applied{}, "", err
}
w := WaitsOf(entries)
if len(w.Ready) == 0 {
return Applied{}, ix.waitingLine(w), nil
}
if !start {
for _, e := range w.Ready {
if e.Value.Taking != "" {
ix.logf("take-in: %s.%s is claimed by the take-in %s, so this indexer takes nothing in", PendingState, e.Key, e.Value.Taking)
return Applied{}, "claimed by another take-in", nil
}
}
}
// Of each repository's merges only the newest is read (from memory, else asked of the forge bridge: a pending value
// keeps a reference); the older ones are covered by it. A newest merge that cannot be read is said and
// holds its revision: nothing of it is taken in silently.
ready, lost, reasons := ix.loadNewest(ctx, w.Ready)
if len(lost) > 0 {
return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons)
}
names, dry, err := ix.computeOver(ctx, ready)
if err != nil {
return Applied{}, "", err
}
if names == own {
a, err := ix.takeInWhole(ctx, ready, own, "")
if errors.Is(err, errClaimLost) {
return a, "claimed by another take-in", nil
}
return a, ix.waitingLine(w), err
}
return Applied{}, ix.waitingLine(w), ix.holdFor(ctx, ready, names, own, dry)
}
// holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds
// it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry
// (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or
// indexer_drop_pending drops it. Nothing of the revision is taken in meanwhile.
func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, reasons map[string]string) error {
t := revisionWait(takein.TakeInPending, ready, "")
for _, g := range lost {
t.Lost = append(t.Lost, g.Key)
}
said := true
for _, e := range ready {
isLost := slices.Contains(t.Lost, e.Key)
if e.Value.Said != t.Key || e.Value.SaidKind != takein.TakeInPending || (isLost && e.Value.Lost == "") {
said = false
}
}
if said {
return nil
}
for _, g := range lost {
ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; a newer merge of %s covers it, indexer_drop_pending drops it",
g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], g.Forge.Repository)
}
if !ix.sayTakeIn(ctx, t) {
return nil
}
var kept []PendingEntry
for _, e := range ready {
if !slices.Contains(t.Lost, e.Key) {
kept = append(kept, e)
}
}
for _, g := range lost {
reason := reasons[g.Key]
ix.markSaid(ctx, []PendingEntry{g}, &t, func(p *Pending) { p.Lost = reason })
}
ix.markSaid(ctx, kept, &t, nil)
return nil
}
// waitingLine says what waits after an evaluation, for the heard state.
func (ix *Indexer) waitingLine(w Waits) string {
var parts []string
for _, g := range slices.Sorted(maps.Keys(w.Groups)) {
parts = append(parts, fmt.Sprintf("the group %q gathered (%d held)", g, len(w.Groups[g])))
}
for _, g := range slices.Sorted(maps.Keys(w.Behind)) {
parts = append(parts, fmt.Sprintf("%d merges behind %s", len(w.Behind[g]), g))
}
return strings.Join(parts, "; ")
}
// dryRun is an indexer computing a take-in without writing it, and the overlays of what it would have written.
type dryRun struct {
ix *Indexer
modules, graph, verdicts *overlayStore
candidates, taken, predefined, configs *overlayStore
oldSections *overlayStore
}
// dry is an indexer over overlays of this one's states (overlay.go): what a take-in would write stays in memory and
// nothing reaches the bus but the manifests and the packages the revision publishes, which clause 1 writes all the
// same. It says no event, kicks nothing in the background, and knows what this one knows of the source, the toolchains
// and the previews. Every field of Indexer is named here (TestDryNamesEveryField holds it so).
func (ix *Indexer) dry() *dryRun {
over := func(st Store) (Store, *overlayStore) {
if st == nil {
return nil, nil
}
o := newOverlay(st)
return o, o
}
d := &dryRun{}
x := &Indexer{Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, Read: ix.Read, Contains: ix.Contains,
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil}
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
x.Store, d.candidates = over(ix.Store)
x.Modules, d.modules = over(ix.Modules)
x.Graph, d.graph = over(ix.Graph)
x.Verdicts, d.verdicts = over(ix.Verdicts)
x.Taken, d.taken = over(ix.Taken)
x.Predefined, d.predefined = over(ix.Predefined)
x.ConfigCandidates, d.configs = over(ix.ConfigCandidates)
x.OldSections, d.oldSections = over(ix.OldSections)
ix.mu.Lock()
x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)}
x.previews = maps.Clone(ix.previews)
x.toolchains = map[string]map[string]delivery.Toolchain{}
for l, by := range ix.toolchains {
x.toolchains[l] = maps.Clone(by)
}
x.moving = maps.Clone(ix.moving)
x.untaken = maps.Clone(ix.untaken)
ix.mu.Unlock()
ix.ownMu.Lock()
x.ownFP = ix.ownFP
ix.ownMu.Unlock()
d.ix = x
return d
}
// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees
// govern (ADR 0633 §3). It answers the fingerprint the indexer's own module has once they are applied (none is "",
// with its reason), and the dry run, whose overlays hold what the take-in would write. A merge that can never be read
// is left out, as its take-in will drop it.
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) {
d := ix.dry()
for _, e := range ready {
if e.Covered {
continue
}
if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil {
if errors.Is(err, ErrUnreadable) {
continue
}
return "", d, fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
}
}
key, _ := delivery.ModulesKey(delivery.IndexerModule)
value, _, found, err := d.ix.Modules.Get(ctx, key)
if err != nil {
return "", d, err
}
if !found {
return NoBuild + "the indexer holds no record of itself", d, nil
}
m, err := delivery.DecodeModule(value)
if err != nil {
return "", d, nil
}
if m.FP2 == "" {
return NoBuild + m.None2, d, nil
}
return m.FP2, d, nil
}
// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another
// take-in wrote the key, or the store failed) releases every claim this take-in made and answers false: nothing is
// taken in, and no claim of this one is left behind.
func (ix *Indexer) claim(ctx context.Context, entries []PendingEntry, own, by string) ([]PendingEntry, bool) {
id := fmt.Sprintf("%s at %s", own, ix.now().Format(time.RFC3339Nano))
if by != "" {
id = by + ": " + id
}
claimed := make([]PendingEntry, 0, len(entries))
for _, e := range entries {
p := e.Value
p.Taking, p.LastWriter = id, own
data, err := encodePending(p)
if err == nil {
err = ix.Pending.Update(ctx, e.Key, data, e.Revision)
}
if err != nil {
ix.logf("take-in: the claim of %s.%s is lost (%v): another take-in took it, so this one takes nothing in", PendingState, e.Key, err)
ix.release(ctx, claimed)
return nil, false
}
e.Value = p
_, rev, _, err := ix.Pending.Get(ctx, e.Key)
if err != nil {
ix.logf("take-in: the claim of %s.%s cannot be read back (%v), so this take-in takes nothing in", PendingState, e.Key, err)
ix.release(ctx, append(claimed, e))
return nil, false
}
e.Revision = rev
claimed = append(claimed, e)
}
return claimed, true
}
// release clears this take-in's claim on each entry, so a later merge, a restart or a hand take-in takes it in. It
// reads the key again and clears it by compare-and-set only while the claim is still this take-in's: a key another
// take-in wrote since, or removed, is left to it. A release that fails is said; the start of the next indexer voids
// the claim (TakeInAtStart).
func (ix *Indexer) release(ctx context.Context, claimed []PendingEntry) {
for _, e := range claimed {
value, rev, found, err := ix.Pending.Get(ctx, e.Key)
if err == nil && !found {
continue
}
var cur Pending
if err == nil {
err = json.Unmarshal(value, &cur)
}
if err == nil && cur.Taking != e.Value.Taking {
continue
}
if err == nil {
cur.Taking = ""
var data []byte
if data, err = encodePending(cur); err == nil {
err = ix.Pending.Update(ctx, e.Key, data, rev)
}
}
if err != nil {
ix.logf("take-in: the claim of %s.%s is not released (%v): the next indexer's start voids it", PendingState, e.Key, err)
}
}
}
// takeInWhole takes the entries in with this indexer, as merges are taken in (ApplyMessage), oldest first, then
// removes them from pending and says each wait they were said under taken. by names a hand act ("" for none).
func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own, by string) (Applied, error) {
return ix.takeInWholeBy(ctx, entries, own, by, "", "")
}
// takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events.
func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) {
claimed, ok := ix.claim(ctx, entries, own, by)
if !ok {
return Applied{}, errClaimLost
}
var applied Applied
newest := map[string]PendingEntry{}
for _, e := range claimed {
if !e.Covered {
newest[strings.ToLower(e.Forge.Repository)] = e
}
}
for _, e := range claimed {
if e.Covered {
by := newest[strings.ToLower(e.Forge.Repository)]
ix.logf("take-in: %s (%s#%d) is covered by %s, the newest merge of %s in this revision: its message is not read, and it is ended with it", e.Key, e.Forge.Repository, e.Forge.Number, by.Key, e.Forge.Repository)
// Its pull request leaves its group's candidate as a merged message takes it out; nothing else of it is applied.
if err := ix.removeCandidate(ctx, e.Forge); err != nil {
ix.release(ctx, claimed)
return applied, fmt.Errorf("taking %s (%s#%d) out of its candidate: %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
}
continue
}
a, err := ix.ApplyMessage(ctx, e.Forge)
applied.Written = append(applied.Written, a.Written...)
if a.Late != "" {
applied.Late = a.Late
}
if errors.Is(err, ErrUnreadable) {
ix.logf("take-in: %s (%s#%d) can never be read, dropped: %v", e.Key, e.Forge.Repository, e.Forge.Number, err)
continue
}
if err != nil {
ix.release(ctx, claimed)
return applied, fmt.Errorf("taking %s (%s#%d) in: %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
}
}
for _, e := range claimed {
if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil {
ix.logf("take-in: %s.%s is taken in and not removed: %v", PendingState, e.Key, err)
}
}
ix.forget(claimed...)
ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s", len(claimed), own, byWords(by), claimed[0].Key)
for _, t := range saidWaits(claimed) {
t.By, t.Why = who, why
ix.sayTakeIn(ctx, t)
}
return applied, nil
}
func byWords(by string) string {
if by == "" {
return ""
}
return " (a hand act: " + by + ")"
}
// saidWaits are the taken events of every wait the entries were said under that is not ended yet.
func saidWaits(entries []PendingEntry) []takein.TakeIn {
by := map[string][]PendingEntry{}
for _, e := range entries {
if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending {
by[e.Value.Said] = append(by[e.Value.Said], e)
}
}
var out []takein.TakeIn
for _, key := range slices.Sorted(maps.Keys(by)) {
held := by[key]
if strings.HasPrefix(key, "g.") {
out = append(out, groupWait(takein.TakeInTaken, held[0].Value.Group, held))
continue
}
t := revisionWait(takein.TakeInTaken, held, held[0].Value.WaitsFor)
t.Key = key
out = append(out, t)
}
return out
}
// holdFor keeps the ready merges pending for the indexer build they name and writes only what building and moving
// that indexer needs (ADR 0633 §1): the indexer's own module entry, its key of the module graph and the verdict on its
// manifest, from the dry run; the manifests and the packages are written by the dry run itself. The revision is said
// pending under its oldest merge's key, and a wait it was said under before by another key is said taken.
func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own string, d *dryRun) error {
key, _ := delivery.ModulesKey(delivery.IndexerModule)
if err := copyKey(ctx, d.modules, ix.Modules, key); err != nil {
return fmt.Errorf("the indexer's own module entry: %w", err)
}
if gk, err := graph.Key(delivery.IndexerModule); err == nil {
if err := copyKey(ctx, d.graph, ix.Graph, gk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the indexer's key of the module graph: %w", err)
}
}
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
if m, err := delivery.DecodeModule(value); err == nil && m.Manifest != "" {
if vk, err := delivery.VerdictsKey(m.Manifest); err == nil {
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
}
}
}
}
t := revisionWait(takein.TakeInPending, ready, names)
var ended, unmarked []PendingEntry
said, changed := false, false
for _, e := range ready {
switch {
case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending:
said = true
changed = changed || e.Value.WaitsFor != names
if e.Value.WaitsFor != names {
unmarked = append(unmarked, e)
}
case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending:
ended = append(ended, e)
unmarked = append(unmarked, e)
default:
unmarked = append(unmarked, e)
}
}
for _, old := range saidWaits(ended) {
ix.sayTakeIn(ctx, old)
}
if !said || changed {
ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own)
if !ix.sayTakeIn(ctx, t) {
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own })
return nil
}
unmarked = ready
}
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own })
return nil
}
// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it.
func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error {
if from == nil || to == nil {
return nil
}
value, deleted, ok := from.written(key)
if !ok {
return nil
}
return write(ctx, to, key, func(cur []byte, found bool) ([]byte, bool, error) {
switch {
case deleted:
return nil, found, nil
case found && string(cur) == string(value):
return nil, false, nil
}
return value, false, nil
})
}
// PendingWaits is what indexer_pending answers of the pending state's entries (ADR 0633 §5): the ready merges as one
// revision, each group being gathered, and the merges behind a group, each with the modules it holds back and the
// last take-in event said of it.
func PendingWaits(entries []PendingEntry) []takein.PendingWait {
w := WaitsOf(entries)
var out []takein.PendingWait
if len(w.Ready) > 0 {
out = append(out, waitOf(w.Ready, takein.TakeInRevision, ""))
}
for _, g := range slices.Sorted(maps.Keys(w.Groups)) {
out = append(out, waitOf(w.Groups[g], takein.TakeInGroup, ""))
}
for _, g := range slices.Sorted(maps.Keys(w.Behind)) {
out = append(out, waitOf(w.Behind[g], takein.TakeInRevision, g))
}
slices.SortStableFunc(out, func(a, b takein.PendingWait) int { return a.Since.Compare(b.Since) })
return out
}
func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
pw := takein.PendingWait{Key: entries[0].Key, What: what, Merges: mergesOf(entries), Behind: behind, Since: sinceOf(entries),
WaitsFor: entries[len(entries)-1].Value.WaitsFor, Holds: []string{}}
if what == takein.TakeInGroup {
pw.Group = entries[0].Value.Group
pw.Key = takein.GroupTakeInKey(pw.Group)
pw.Members = entries[len(entries)-1].Forge.Members
pw.WaitsFor = ""
}
for _, e := range entries {
pw.Keys = append(pw.Keys, e.Key)
for _, name := range e.Holds {
if !slices.Contains(pw.Holds, name) {
pw.Holds = append(pw.Holds, name)
}
}
if e.Value.Lost != "" {
pw.Lost = append(pw.Lost, e.Key)
}
if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) {
pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt
}
}
slices.Sort(pw.Holds)
return pw
}
// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5): the wait named by its key, the
// ready revision (its m. key) or a group gathered so far (its g. key), is taken in with this indexer and its judge,
// whatever build it names, for when the named indexer can never run or the group's end never comes. by and why are
// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after.
func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn {
r := TakenIn{Module: key}
if ix.Pending == nil {
r.Answer = "the pending state is not on the bus: nothing waits"
return r
}
own := ix.ownBuildNow(ctx)
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
r.Answer = err.Error()
return r
}
w := WaitsOf(entries)
var take []PendingEntry
switch {
case strings.HasPrefix(key, "g."):
for g, held := range w.Groups {
if takein.GroupTakeInKey(g) == key {
take = held
}
}
case len(w.Ready) > 0 && w.Ready[0].Key == key:
take = w.Ready
}
if len(take) == 0 {
r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key)
return r
}
take, lost, reasons := ix.loadNewest(ctx, take)
if len(lost) > 0 {
r.Answer = fmt.Sprintf("nothing is taken in: the message of %s is not read (%s); a newer merge of its repository covers it, indexer_drop_pending drops it", lost[0].Key, reasons[lost[0].Key])
return r
}
ix.logf("take-in: a hand take-in of %s (%d merges) asked by %s: %s", key, len(take), by, why)
hand := fmt.Sprintf("%s (%s)", by, why)
if _, err := ix.takeInWholeBy(ctx, take, own, hand, by, why); err != nil {
r.Answer = err.Error()
return r
}
r.Stored = true
r.Answer = fmt.Sprintf("%d merges taken in by the running indexer %s", len(take), own)
if _, _, err := ix.evaluate(ctx, own, false); err != nil {
r.Answer += "; what waited behind it is not evaluated: " + err.Error()
}
return r
}
// DropPending is the hand drop (indexer_drop_pending, ADR 0633 §5): the one pending merge under key, abandoned
// because a later merge replaced it, is removed and nothing of it is taken in. Its wait is said taken, naming the drop;
// the merges left under that wait are said pending again by the evaluation that follows.
func (ix *Indexer) DropPending(ctx context.Context, key, by, why string) TakenIn {
r := TakenIn{Module: key}
if ix.Pending == nil {
r.Answer = "the pending state is not on the bus: nothing waits"
return r
}
own := ix.ownBuildNow(ctx)
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
r.Answer = err.Error()
return r
}
at := slices.IndexFunc(entries, func(e PendingEntry) bool { return e.Key == key })
if at < 0 {
r.Answer = fmt.Sprintf("%s holds no merge under %s", PendingState, key)
return r
}
e := entries[at]
if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil {
r.Answer = fmt.Sprintf("%s.%s is not removed: %v", PendingState, key, err)
return r
}
ix.forget(e)
ix.logf("take-in: %s (%s#%d at %s) dropped by hand, asked by %s: %s; nothing of it is taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge, by, why)
if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending {
var same []PendingEntry
for _, o := range entries {
if o.Key != key && o.Value.Said == e.Value.Said {
same = append(same, o)
}
}
for _, t := range saidWaits([]PendingEntry{e}) {
t.By, t.Why, t.Dropped = by, why, true
ix.sayTakeIn(ctx, t)
}
ix.markSaid(ctx, same, nil, func(p *Pending) { p.Said, p.SaidKind = "", "" })
}
r.Stored, r.Answer = true, fmt.Sprintf("%s dropped: %s#%d at %s is not taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge)
if own != "" {
if _, _, err := ix.evaluate(ctx, own, false); err != nil {
r.Answer += "; what waits is not evaluated again: " + err.Error()
}
}
return r
}
// TakeInMerge is the hand take-in of one merge (indexer_take_in_merge, issue 1585): its merged message is asked of the forge
// bridge and handed to the take-in as a delivered merged message is (Arrive, under the stream sequence seq, which keeps the
// merge order), so it is held for the indexer its revision names or taken in, and a merge already held under that
// sequence is not written twice. For a merge the take-in never took in. The bridge's own words answer when it cannot give
// the message.
func (ix *Indexer) TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) TakenIn {
r := TakenIn{Module: takein.PendingKey(seq)}
if ix.Message == nil {
r.Answer = "this indexer does not ask the forge bridge"
return r
}
data, err := ix.Message(ctx, repository, merge)
if err != nil {
r.Answer = fmt.Sprintf("the forge bridge could not answer for the merge of %s at %s: %v", repository, merge, err)
return r
}
f, err := forgemsg.DecodeForge(data)
if err != nil {
r.Answer = fmt.Sprintf("the forge bridge's answer for %s at %s is not a forge message: %v", repository, merge, err)
return r
}
if f.Kind != forgemsg.ForgeMerged || f.Merge != merge || f.Repository != repository {
r.Answer = fmt.Sprintf("the forge bridge's answer is not the merge of %s at %s", repository, merge)
return r
}
ix.logf("take-in: a hand take-in of the merge %s#%d at %s (sequence %d) asked by %s: %s", f.Repository, f.Number, f.Merge, seq, by, why)
a, words, err := ix.Arrive(ctx, seq, f)
if err != nil {
r.Answer = err.Error()
return r
}
r.Stored, r.Repository, r.Commit = true, f.Repository, f.Merge
r.Answer = fmt.Sprintf("%s#%d at %s handed to the take-in as a delivered merge (%d keys written)", f.Repository, f.Number, f.Merge, len(a.Written))
if words != "" {
r.Answer += "; " + words
}
return r
}