Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/candidates.go
T
jschoubben 310d4817ca indexer: take-in.taken names its branch, source revision and entries; runner: one with no entries starts nothing
The pipeline runner heard every take-in.taken with branch "" and source 0 and started no pipeline, asking the bus
for each again for ever: the indexer's event body never gained design 47 G §476, §493 and §648's fields. Every taken
event of one take-in now names the watched branch, the source revision it makes (one above the newest seen or said,
A-431) and one entry per module written; a taken event with no entries starts nothing and is journaled (§493).
2026-11-15 17:50:46 +01:00

575 lines
32 KiB
Go

package index
import (
"bytes"
"cmp"
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"reflect"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// ErrConflict is what a Store says when a write found the key changed or created since it was read.
var ErrConflict = errors.New("the key changed since it was read")
// ErrUnreadable wraps what says a message cannot be indexed, ever: it is never tried again.
var ErrUnreadable = errors.New("unreadable message")
// Store is the candidates bucket: a compare-and-set key-value store.
type Store interface {
// Get is the key's value and revision; found is false when there is none.
Get(ctx context.Context, key string) (value []byte, revision uint64, found bool, err error)
// Create writes a key that must not exist (ErrConflict when it does).
Create(ctx context.Context, key string, value []byte) error
// Update writes the key if its revision is still revision (ErrConflict when not).
Update(ctx context.Context, key string, value []byte, revision uint64) error
// Delete removes the key if its revision is still revision (ErrConflict when not).
Delete(ctx context.Context, key string, revision uint64) error
// Keys is every key the store holds now; none is an empty list, not an error.
Keys(ctx context.Context) ([]string, error)
}
// trunkLike are the branches that are no group: a pull request from one is nobody's delivery group (the rule of
// mesh-catalog modules/mesh-delivery/cmd/mesh-delivery/holder.go:321).
var trunkLike = map[string]bool{TrunkBranch: true, "master": true, "trunk": true, "": true}
// writeTries bounds the reads again after a conflict; a message that still conflicts is left to the server to deliver again.
const writeTries = 8
// Indexer writes candidates.<group> from the forge bridge's messages.
type Indexer struct {
// BranchHead is the head of a repository's branch, asked of the seat git's verb branch: TakeIn reads a module a
// registration names at its first watched branch's head (design 47 G A-85: the trunk is the first watched branch's).
BranchHead func(ctx context.Context, repository, branch string) (string, error)
// keptView replaces KeepView for a test that counts the writes of the view; nil in the program.
keptView func(context.Context) error
// Module is the indexer's own module, the credential's: the observer of its findings and the key of its own entry (design 47 G A-296).
Module string
Store Store // candidates
// Modules is the modules bucket; nil writes no fingerprints.
Modules Store
// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
Manifests Store
// Predefined and ConfigCandidates are the control records' states (control_records.go, ADR 0565 §2); nil writes none.
Predefined, ConfigCandidates Store
// Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none.
Taken Store
// Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing.
Verdicts Store
// OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no
// count and the judge refuses no use of an old section.
OldSections Store
// Unjudged is the state of the manifest digests a take-in wrote and no judge pass judged yet (unjudged.go, issue 1966);
// nil keeps them only in the take-in's memory.
Unjudged Store
// Requirements is the requirements state (requirements.go, DLV-I7): what each consumer's go.sum requires of each mesh
// Go module, kept at each merge; nil keeps none.
Requirements Store
reqMu sync.Mutex
reqKept map[string]bool // consumers whose go.sum this process kept (requirementsKept)
// JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's.
JudgeID string
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
SayRefused func(ctx context.Context, r meshcatalog.ManifestRefusedEvent)
// SayAccepted publishes the event manifest-accepted a under the bus's duplicate id id (accepted.go; issue 1966 D3,
// DLV-I9): for every accepted verdict newly written, carrying the entries whole, and an empty one for a module leaving
// the trunk; nil says nothing.
SayAccepted func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error
// Findings is the indexer's finding writer (design 47 G A-29), where a manifest-accepted whose publish failed twice is kept
// until said (accepted.go, DLV-I9b); nil writes none.
Findings FindingWriter
// Packages is the packages state (PackagesState, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// and every fingerprint takes the version line.
Packages Store
// SeatDefinitions is the state seat-definitions (seatdefs.go; design 26, DLV-I17); nil keeps none.
SeatDefinitions Store
// ReadFile reads one file of a repository at a commit (the seat git's file); a file the commit does not hold is
// nil with no error. Nil reads no seat definition.
ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error)
// SaySeatDefinition says seat-definition-changed (seatdefs.go); nil says nothing.
SaySeatDefinition func(ctx context.Context, c SeatDefinitionChanged) error
// Revisions is the object bucket catalog-revisions (catalog_revisions.go; design 47, Two graphs); nil keeps no
// catalog graph revision.
Revisions Objects
// EntryIndex is the state entry-index (entry_verb.go, design 47 G §675); nil keeps none.
EntryIndex Store
// NamedSets is the state named: which want or running build names a fingerprint (named.go, §675); nil keeps none, and no
// entry is then kept for a name.
NamedSets *Named
// SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing.
SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error
// SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate
// id given (W4.6).
SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error
// Versions is the state versions (versions.go, design 47 G §349): each module's released versions; nil keeps none.
Versions Store
// BuildEdges is the state build-edges (branch_builds.go, design 47 G §374): the edge entries of the newest build of a module on a
// watched branch other than the first; nil keeps none.
BuildEdges Store
// SayVersionReleased says the seat mesh-catalog's event version-released under the duplicate id given (§349); nil says nothing.
SayVersionReleased func(ctx context.Context, e meshcatalog.VersionReleasedEvent, id string) error
// SayVersionsFilled says the seat mesh-catalog's event versions-filled, once, under the duplicate id given (§372); nil says nothing.
SayVersionsFilled func(ctx context.Context, e meshcatalog.VersionsFilledEvent, id string) error
// Inputs is the state inputs (run_inputs.go, design 47 G §354): the inputs of the indexer's run asks; nil keeps none.
Inputs Store
// InputObjects is the object store beside the state inputs (`inputs-objects`): an input too large for a key is kept there
// (mesh-sdk stateobjects, design 47 G §712, A-237); nil keeps a large input nowhere and refuses it.
InputObjects stateobjects.Objects
// Registry is the registrations and their trusted changes (registrations.go, design 47 G §2 to §4, §667); SeedModules reads
// what the first registrations are composed from (§8). Nil: no registration act is answered.
Registry *Registry
SeedModules func(ctx context.Context) ([]SeedModule, error)
// Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph.
Graph Store
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
// with its git object id (the seat git's trees with files): the listing a package's digest is computed from.
// Nil publishes no package.
Files func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error)
// GoSum reads a module directory's go.sum at a commit (the seat git's file): the mesh Go modules a module with no
// reported import set takes the version line of. Nil reads none: such a module keeps what it held.
GoSum func(ctx context.Context, repository, sha, dir string) ([]byte, error)
// GoMod reads a module directory's go.mod at a commit (the seat git's file): the mesh Go modules of its own
// repository it replaces by a directory (fingerprint.ReplacedSiblings), which the version line takes by their
// content; nil reads none.
GoMod func(ctx context.Context, repository, sha, dir string) ([]byte, error)
// Now is the clock of a taken value; nil is time.Now.
Now func() time.Time
Builds *Builds
// Named asks the forge for the git object of each named path of the module directories given, at a commit
// (the seat git's trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a
// manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths
// has none.
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
Trunk func(ctx context.Context, repository, sha string) ([]string, error)
// Read asks the forge for a repository's trees at a commit and the manifest and lock-file hashes of the one module directory dir (the
// root spelled ""), as the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module.
Read func(ctx context.Context, repository, sha, dir string) (RepoRead, error)
// Contains says whether a commit is the second or an ancestor of it (the seat git's contains).
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
Logf func(format string, args ...any)
// ReadOpen asks the forge which pull requests into the trunk are open now (the seat git's open-pulls). A catalogue pull message
// that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error)
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource)
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
freshMu sync.Mutex
recompute sync.Mutex // one Recompute at a time
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
OwnBuild func(ctx context.Context) (string, error)
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
ownMu sync.Mutex // guards ownFP
keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries)
// Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the
// events stream has not been given: a revision is removed only while none is pending (§579).
Pins Store
PinsPending func(ctx context.Context) (uint64, error)
namesMu sync.Mutex // packageNames
packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go)
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
ownWhy string
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
OwnBuildState Store
// Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since),
// toolchain.<language>, delivered.<module>, pending.<module>.<version>, acted.<acting part>.<dependent>, each key part escaped as a name
// (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled.
Heard Store
// Placement asks the assigner's verb placement (A-62) and RunningWhole reads the node-engine's stream running whole (§556), both for the
// edge entries of the first fill (§377); nil reads none, and the entries of kinds 2, 4 and 6 then read "not yet delivered".
Placement PlacementAsker
RunningWhole RunningReader
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
// Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a
// pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none.
Message func(ctx context.Context, repository, merge string, number int) ([]byte, error)
heldMu sync.Mutex
held map[uint64]forgemsg.Forge // the merged messages Arrive was handed, by stream sequence, until their merge is taken in or dropped
// Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil
// takes every merge in at once, as before the take-in step.
Pending Store
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
// ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker,
// A-248); nil validates nothing, and a revision is taken in as before.
ValidationAsks ValidationAsks
// AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an
// object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate.
AskVerb buildstep.AskVerb
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
Runs Store
runMu sync.Mutex // one run event at a time (runcandidates.go)
pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go)
}
func (ix *Indexer) logf(format string, args ...any) {
if ix.Logf != nil {
ix.Logf(format, args...)
}
}
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
// merged message also writes the fingerprint of each module of its repository (modules.go).
func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error {
_, err := ix.ApplyMessage(ctx, f)
return err
}
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) {
ix.kickJudge(ctx)
var applied Applied
if f.Kind == forgemsg.ForgeOpen {
return applied, ix.applyOpen(ctx, f)
}
if f.Kind != forgemsg.ForgePull && f.Kind != forgemsg.ForgeMerged {
return applied, nil
}
lateMerge := false
if f.Kind == forgemsg.ForgeMerged && ix.Modules != nil {
w, err := ix.applyMerged(ctx, f)
applied = Applied{Written: w.written, Late: w.late}
if err != nil {
return applied, err
}
ix.logf("%s", w.said(f.Number))
if w.late == "" { // a late message leaves the taken key, which names a later merge
if err := ix.writeTakenKnown(ctx, f, w.asked); err != nil {
return applied, err
}
// The control records (ADR 0565 §2): the catalogue's predefined list, and the open pull requests' config
// declarations judged again against the trunk this merge moved.
if err := ix.writePredefined(ctx, f); err != nil {
return applied, err
}
if err := ix.configCandidatesAfterMerge(ctx, f, writtenModules(w.written)); err != nil {
return applied, err
}
// Each package's name in its own ecosystem, which the builder numbers it under (design 32), kept before the
// manifests are judged: a manifest of this merge naming a package this merge brings is judged knowing its name.
if err := ix.KeepPackageNames(ctx, f); err != nil {
return applied, err
}
// Every manifest judged again: a rule between manifests can turn on the one this merge moved (ADR 0578 §1).
// What this take-in newly wrote is fresh: judged in full (ADR 0579 §1).
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
// Judged again at the next message or source revision heard (kickJudge), as a failed start pass is.
ix.judgePending.Store(true)
ix.logf("judge: the manifests are not judged after %s#%d: %v; judged again at the next message heard", f.Repository, f.Number, err)
}
}
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
lateMerge = w.late != ""
}
// The seat definitions of the contracts modules it carries (DLV-I17, I18): a late merge moves no trunk's.
if !lateMerge {
if err := ix.ApplySeatDefinitions(ctx, f); err != nil {
return applied, err
}
}
var digests map[string]string
var held map[string]meshcatalog.ModulesByModuleValue
if f.Kind == forgemsg.ForgePull {
var err error
if digests, err = ix.keepManifests(ctx, f); err != nil {
return applied, err
}
if held, err = ix.heldModules(ctx); err != nil {
return applied, err
}
// Each manifest the pull carries is judged by this judge, unless it holds a verdict already that this judge made or
// that accepts it: a refusal an older judge made never stands for a candidate (or a digest back on the trunk).
over := map[string]string{}
for _, digest := range digests {
if ix.Manifests == nil {
break
}
if value, _, found, err := ix.Manifests.Get(ctx, digest); err == nil && found {
if kept, err := meshcatalogclient.DecodeManifest(value); err == nil {
over[kept.Module] = digest
}
}
}
ix.judgeDigests(ctx, over, fmt.Sprintf("%s#%d", f.Repository, f.Number))
if err := ix.configCandidateOfPull(ctx, f, held); err != nil {
if !errors.Is(err, ErrUnreadable) {
return applied, err
}
ix.logf("%s#%d: its config declarations are not written: %v", f.Repository, f.Number, err)
}
}
return applied, ix.writeCandidate(ctx, f, digests, held)
}
// removeCandidate takes a merged pull request out of its group's candidate and writes nothing else of its message: for a
// merge a newer merge of the same repository covers (it needs only the repository, number and branch).
func (ix *Indexer) removeCandidate(ctx context.Context, f forgemsg.Forge) error {
return ix.writeCandidate(ctx, f, nil, nil)
}
// writeCandidate writes the group's candidate with the pull request f names (taken out for a merged or closed one).
func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) error {
if trunkLike[f.Branch] {
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
return nil
}
key, err := CandidatesKey(f.Branch)
if err != nil {
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
}
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
var cur *meshcatalog.CandidatesByGroupValue
if found {
c, err := DecodeCandidate(value)
if err != nil {
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
}
cur = &c
}
next := ix.next(cur, f, digests, held)
if next != nil {
ix.withPreviews(ctx, next)
if err := ix.refuseDisagreement(*next); err != nil {
return nil, false, err
}
}
switch {
case next == nil && cur == nil, next != nil && cur != nil && SameCandidate(*cur, *next):
return nil, false, nil
case next == nil:
return nil, true, nil
}
data, err := EncodeCandidate(*next)
return data, false, err
})
}
// applyOpen takes an open message into the candidates (novox/hq ADR 0443 §5): every pull request it does not name is
// taken out of its group, and a group left with none is deleted. What the message names is left as it is. A pull request
// opened after the bridge read the forge is heard after this message and added again by its own.
func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error {
named := make(map[forgemsg.OpenPull]bool, len(f.Open))
for _, p := range f.Open {
named[p] = true
}
keys, err := ix.Store.Keys(ctx)
if err != nil {
return fmt.Errorf("listing the candidates: %w", err)
}
for _, key := range keys {
if meshcatalogclient.IsTryGroup(key) {
continue // a run's candidate is no pull request's: its run owns it (runcandidates.go)
}
err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
c, err := DecodeCandidate(value)
if err != nil {
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
}
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p meshcatalog.CandidatesByGroupValuePullsItem) bool {
return !named[forgemsg.OpenPull{Repository: p.Repository, Number: int(p.Number)}]
})
switch {
case len(kept) == len(c.Pulls):
return nil, false, nil
case len(kept) == 0:
ix.logf("%s: no pull request of the group is open, so no candidate", key)
return nil, true, nil
}
heads := make([]meshdeliveryclient.Head, len(kept))
for i, p := range kept {
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
}
hash, err := meshdeliveryclient.HeadsHash(heads)
if err != nil {
return nil, false, fmt.Errorf("%w: candidates.%s: %v", ErrUnreadable, key, err)
}
c.Pulls, c.Heads = kept, hash
data, err := EncodeCandidate(c)
return data, false, err
})
if err != nil {
return err
}
}
return nil
}
// write changes one key of a store by compare-and-set: compute is given the value read (found says there is one) and
// answers the new value, or del to remove the key, or neither for no change. A conflict reads again, up to writeTries.
// read, when given, is set to the revision of the value compute was last given.
//
// **A value whose bytes are the bytes held is not written** (issue 1966, 2026-11-08: a compare of two map types that never
// matched rewrote every module on every recompute, each write a views event, and a reader of the views took each as a gap
// and asked its whole first view again): whatever compute decides, a write that changes nothing but the revision is none.
func write(ctx context.Context, st Store, key string, compute func(value []byte, found bool) (next []byte, del bool, err error), read ...*uint64) error {
for try := 0; try < writeTries; try++ {
value, revision, found, err := st.Get(ctx, key)
if err != nil {
return fmt.Errorf("reading %s: %w", key, err)
}
next, del, err := compute(value, found)
if err != nil {
return err
}
for _, r := range read {
*r = revision
}
switch {
case next == nil && !del, unchanged(next, del, value, found):
return nil
case del:
err = st.Delete(ctx, key, revision)
case !found:
err = st.Create(ctx, key, next)
default:
err = st.Update(ctx, key, next, revision)
}
if errors.Is(err, ErrConflict) {
continue
}
if err != nil {
return fmt.Errorf("writing %s: %w", key, err)
}
return nil
}
return fmt.Errorf("%s changed under every one of %d tries", key, writeTries)
}
// unchanged says that compute's answer next is the value held: write writes nothing then (write).
func unchanged(next []byte, del bool, held []byte, found bool) bool {
return found && !del && next != nil && bytes.Equal(next, held)
}
// next is the group after the message; nil when the group is left with no pull request.
//
// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of
// its manifest and why it has no fingerprint of the new form (meshcatalogclient.NoneNoPreview), and the candidate its FPSet (issue 1451); a
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir).
func (ix *Indexer) next(cur *meshcatalog.CandidatesByGroupValue, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) *meshcatalog.CandidatesByGroupValue {
var pulls []meshcatalog.CandidatesByGroupValuePullsItem
var base int64
if cur != nil {
pulls, base = slices.Clone(cur.Pulls), cur.Base
}
at := slices.IndexFunc(pulls, func(p meshcatalog.CandidatesByGroupValuePullsItem) bool {
return p.Repository == f.Repository && p.Number == int64(f.Number)
})
if f.Kind == forgemsg.ForgeMerged || f.Action == forgemsg.PullClosed {
if at < 0 {
return cur
}
pulls = slices.Delete(pulls, at, at+1)
} else {
entry := meshcatalog.CandidatesByGroupValuePullsItem{Repository: f.Repository, Number: int64(f.Number), Head: f.Head, Modules: []string{}}
switch {
case f.DirsKnown:
for _, dir := range f.Dirs {
m := moduleOfDir(held, f, dir)
if !slices.Contains(entry.Modules, m) {
entry.Modules = append(entry.Modules, m)
}
if digest, ok := digests[dir]; ok {
if entry.Fps == nil {
entry.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
}
entry.Fps[m] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: digest, None2: meshcatalogclient.NoneNoPreview}
}
}
case at >= 0 && pulls[at].Head == f.Head:
// Its directories are not known now, for the head the group already holds: what it knew stands.
entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps
}
if at >= 0 {
pulls[at] = entry
} else {
pulls = append(pulls, entry)
}
}
if len(pulls) == 0 {
return nil
}
slices.SortFunc(pulls, func(a, b meshcatalog.CandidatesByGroupValuePullsItem) int {
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
return c
}
return cmp.Compare(a.Number, b.Number)
})
heads := make([]meshdeliveryclient.Head, len(pulls))
for i, p := range pulls {
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
}
hash, _ := meshdeliveryclient.HeadsHash(heads)
return &meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: f.Branch, Base: base, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}
}
// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the
// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
// request is judged again.
func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) error {
first := map[string]meshcatalog.CandidatesByGroupValuePullsItem{}
for _, p := range c.Pulls {
for module, m := range p.Fps {
before, ok := first[module]
if !ok {
first[module] = p
continue
}
if !reflect.DeepEqual(before.Fps[module], m) {
err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
return err
}
}
}
return nil
}