The pipeline runner heard every take-in.taken with branch "" and source 0 and started no pipeline, asking the bus for each again for ever: the indexer's event body never gained design 47 G §476, §493 and §648's fields. Every taken event of one take-in now names the watched branch, the source revision it makes (one above the newest seen or said, A-431) and one entry per module written; a taken event with no entries starts nothing and is journaled (§493).
575 lines
32 KiB
Go
575 lines
32 KiB
Go
package index
|
|
|
|
import (
|
|
"bytes"
|
|
"cmp"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
|
"reflect"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// ErrConflict is what a Store says when a write found the key changed or created since it was read.
|
|
var ErrConflict = errors.New("the key changed since it was read")
|
|
|
|
// ErrUnreadable wraps what says a message cannot be indexed, ever: it is never tried again.
|
|
var ErrUnreadable = errors.New("unreadable message")
|
|
|
|
// Store is the candidates bucket: a compare-and-set key-value store.
|
|
type Store interface {
|
|
// Get is the key's value and revision; found is false when there is none.
|
|
Get(ctx context.Context, key string) (value []byte, revision uint64, found bool, err error)
|
|
// Create writes a key that must not exist (ErrConflict when it does).
|
|
Create(ctx context.Context, key string, value []byte) error
|
|
// Update writes the key if its revision is still revision (ErrConflict when not).
|
|
Update(ctx context.Context, key string, value []byte, revision uint64) error
|
|
// Delete removes the key if its revision is still revision (ErrConflict when not).
|
|
Delete(ctx context.Context, key string, revision uint64) error
|
|
// Keys is every key the store holds now; none is an empty list, not an error.
|
|
Keys(ctx context.Context) ([]string, error)
|
|
}
|
|
|
|
// trunkLike are the branches that are no group: a pull request from one is nobody's delivery group (the rule of
|
|
// mesh-catalog modules/mesh-delivery/cmd/mesh-delivery/holder.go:321).
|
|
var trunkLike = map[string]bool{TrunkBranch: true, "master": true, "trunk": true, "": true}
|
|
|
|
// writeTries bounds the reads again after a conflict; a message that still conflicts is left to the server to deliver again.
|
|
const writeTries = 8
|
|
|
|
// Indexer writes candidates.<group> from the forge bridge's messages.
|
|
type Indexer struct {
|
|
// BranchHead is the head of a repository's branch, asked of the seat git's verb branch: TakeIn reads a module a
|
|
// registration names at its first watched branch's head (design 47 G A-85: the trunk is the first watched branch's).
|
|
BranchHead func(ctx context.Context, repository, branch string) (string, error)
|
|
// keptView replaces KeepView for a test that counts the writes of the view; nil in the program.
|
|
keptView func(context.Context) error
|
|
// Module is the indexer's own module, the credential's: the observer of its findings and the key of its own entry (design 47 G A-296).
|
|
Module string
|
|
Store Store // candidates
|
|
// Modules is the modules bucket; nil writes no fingerprints.
|
|
Modules Store
|
|
// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
|
|
Manifests Store
|
|
// Predefined and ConfigCandidates are the control records' states (control_records.go, ADR 0565 §2); nil writes none.
|
|
Predefined, ConfigCandidates Store
|
|
// Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none.
|
|
Taken Store
|
|
// Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing.
|
|
Verdicts Store
|
|
// OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no
|
|
// count and the judge refuses no use of an old section.
|
|
OldSections Store
|
|
// Unjudged is the state of the manifest digests a take-in wrote and no judge pass judged yet (unjudged.go, issue 1966);
|
|
// nil keeps them only in the take-in's memory.
|
|
Unjudged Store
|
|
// Requirements is the requirements state (requirements.go, DLV-I7): what each consumer's go.sum requires of each mesh
|
|
// Go module, kept at each merge; nil keeps none.
|
|
Requirements Store
|
|
reqMu sync.Mutex
|
|
reqKept map[string]bool // consumers whose go.sum this process kept (requirementsKept)
|
|
// JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's.
|
|
JudgeID string
|
|
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
|
|
SayRefused func(ctx context.Context, r meshcatalog.ManifestRefusedEvent)
|
|
// SayAccepted publishes the event manifest-accepted a under the bus's duplicate id id (accepted.go; issue 1966 D3,
|
|
// DLV-I9): for every accepted verdict newly written, carrying the entries whole, and an empty one for a module leaving
|
|
// the trunk; nil says nothing.
|
|
SayAccepted func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error
|
|
// Findings is the indexer's finding writer (design 47 G A-29), where a manifest-accepted whose publish failed twice is kept
|
|
// until said (accepted.go, DLV-I9b); nil writes none.
|
|
Findings FindingWriter
|
|
// Packages is the packages state (PackagesState, packages.go; novox/hq ADR 0593 §2); nil publishes none,
|
|
// and every fingerprint takes the version line.
|
|
Packages Store
|
|
// SeatDefinitions is the state seat-definitions (seatdefs.go; design 26, DLV-I17); nil keeps none.
|
|
SeatDefinitions Store
|
|
// ReadFile reads one file of a repository at a commit (the seat git's file); a file the commit does not hold is
|
|
// nil with no error. Nil reads no seat definition.
|
|
ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error)
|
|
// SaySeatDefinition says seat-definition-changed (seatdefs.go); nil says nothing.
|
|
SaySeatDefinition func(ctx context.Context, c SeatDefinitionChanged) error
|
|
// Revisions is the object bucket catalog-revisions (catalog_revisions.go; design 47, Two graphs); nil keeps no
|
|
// catalog graph revision.
|
|
Revisions Objects
|
|
// EntryIndex is the state entry-index (entry_verb.go, design 47 G §675); nil keeps none.
|
|
EntryIndex Store
|
|
// NamedSets is the state named: which want or running build names a fingerprint (named.go, §675); nil keeps none, and no
|
|
// entry is then kept for a name.
|
|
NamedSets *Named
|
|
// SayCatalogRevision says the seat mesh-catalog's event catalog-revision for a revision kept; nil says nothing.
|
|
SayCatalogRevision func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error
|
|
// SayFill says the seat mesh-catalog's event filling, full or room of the catalog graph revisions, under the duplicate
|
|
// id given (W4.6).
|
|
SayFill func(ctx context.Context, event, id string, f meshcatalog.Fill) error
|
|
// Versions is the state versions (versions.go, design 47 G §349): each module's released versions; nil keeps none.
|
|
Versions Store
|
|
// BuildEdges is the state build-edges (branch_builds.go, design 47 G §374): the edge entries of the newest build of a module on a
|
|
// watched branch other than the first; nil keeps none.
|
|
BuildEdges Store
|
|
// SayVersionReleased says the seat mesh-catalog's event version-released under the duplicate id given (§349); nil says nothing.
|
|
SayVersionReleased func(ctx context.Context, e meshcatalog.VersionReleasedEvent, id string) error
|
|
// SayVersionsFilled says the seat mesh-catalog's event versions-filled, once, under the duplicate id given (§372); nil says nothing.
|
|
SayVersionsFilled func(ctx context.Context, e meshcatalog.VersionsFilledEvent, id string) error
|
|
// Inputs is the state inputs (run_inputs.go, design 47 G §354): the inputs of the indexer's run asks; nil keeps none.
|
|
Inputs Store
|
|
// InputObjects is the object store beside the state inputs (`inputs-objects`): an input too large for a key is kept there
|
|
// (mesh-sdk stateobjects, design 47 G §712, A-237); nil keeps a large input nowhere and refuses it.
|
|
InputObjects stateobjects.Objects
|
|
// Registry is the registrations and their trusted changes (registrations.go, design 47 G §2 to §4, §667); SeedModules reads
|
|
// what the first registrations are composed from (§8). Nil: no registration act is answered.
|
|
Registry *Registry
|
|
SeedModules func(ctx context.Context) ([]SeedModule, error)
|
|
// Graph is the module graph's state (GraphBucket, graph.go; novox/hq ADR 0603 §2); nil writes no graph.
|
|
Graph Store
|
|
// Files asks the forge for every blob under each directory at a commit, by directory then path relative to it, each
|
|
// with its git object id (the seat git's trees with files): the listing a package's digest is computed from.
|
|
// Nil publishes no package.
|
|
Files func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error)
|
|
// GoSum reads a module directory's go.sum at a commit (the seat git's file): the mesh Go modules a module with no
|
|
// reported import set takes the version line of. Nil reads none: such a module keeps what it held.
|
|
GoSum func(ctx context.Context, repository, sha, dir string) ([]byte, error)
|
|
// GoMod reads a module directory's go.mod at a commit (the seat git's file): the mesh Go modules of its own
|
|
// repository it replaces by a directory (fingerprint.ReplacedSiblings), which the version line takes by their
|
|
// content; nil reads none.
|
|
GoMod func(ctx context.Context, repository, sha, dir string) ([]byte, error)
|
|
// Now is the clock of a taken value; nil is time.Now.
|
|
Now func() time.Time
|
|
Builds *Builds
|
|
// Named asks the forge for the git object of each named path of the module directories given, at a commit
|
|
// (the seat git's trees with paths; novox/hq ADR 0499 clauses 2 and 3): the indexer is the one reader of a
|
|
// manifest's paths (ADR 0495 clause 1, the lead's decision L8 of R4e). Nil asks nothing, and a module naming paths
|
|
// has none.
|
|
Named func(ctx context.Context, repository, sha string, paths map[string][]string) (map[string]map[string]string, error)
|
|
Trunk func(ctx context.Context, repository, sha string) ([]string, error)
|
|
// Read asks the forge for a repository's trees at a commit and the manifest and lock-file hashes of the one module directory dir (the
|
|
// root spelled ""), as the forge bridge reads them for a merged message (registered.go). Nil indexes no registered module.
|
|
Read func(ctx context.Context, repository, sha, dir string) (RepoRead, error)
|
|
// Contains says whether a commit is the second or an ancestor of it (the seat git's contains).
|
|
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
|
|
Logf func(format string, args ...any)
|
|
// ReadOpen asks the forge which pull requests into the trunk are open now (the seat git's open-pulls). A catalogue pull message
|
|
// that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a
|
|
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
|
|
ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error)
|
|
|
|
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
|
source sourceNow
|
|
saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource)
|
|
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
|
|
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
|
|
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
|
|
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
|
|
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
|
|
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
|
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
|
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
|
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
|
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
|
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
|
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
|
freshMu sync.Mutex
|
|
recompute sync.Mutex // one Recompute at a time
|
|
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
|
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
|
fillAgain atomic.Bool // a kick arrived while it ran
|
|
atHeads map[string]map[string]meshcatalog.PackagesByModuleValue // a member head's published Go modules, by repository@head (candidatePublished)
|
|
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
|
|
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
|
|
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
|
|
OwnBuild func(ctx context.Context) (string, error)
|
|
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
|
|
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
|
|
ownMu sync.Mutex // guards ownFP
|
|
keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries)
|
|
// Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the
|
|
// events stream has not been given: a revision is removed only while none is pending (§579).
|
|
Pins Store
|
|
PinsPending func(ctx context.Context) (uint64, error)
|
|
namesMu sync.Mutex // packageNames
|
|
packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go)
|
|
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
|
|
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
|
|
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
|
|
ownFixed bool // FixOwnBuild asked already: it asks once
|
|
ownSince time.Time // when the build was asked and not known, with ownWhy
|
|
ownWhy string
|
|
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
|
|
OwnBuildState Store
|
|
// Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since),
|
|
// toolchain.<language>, delivered.<module>, pending.<module>.<version>, acted.<acting part>.<dependent>, each key part escaped as a name
|
|
// (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled.
|
|
Heard Store
|
|
// Placement asks the assigner's verb placement (A-62) and RunningWhole reads the node-engine's stream running whole (§556), both for the
|
|
// edge entries of the first fill (§377); nil reads none, and the entries of kinds 2, 4 and 6 then read "not yet delivered".
|
|
Placement PlacementAsker
|
|
RunningWhole RunningReader
|
|
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
|
// Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a
|
|
// pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none.
|
|
Message func(ctx context.Context, repository, merge string, number int) ([]byte, error)
|
|
heldMu sync.Mutex
|
|
held map[uint64]forgemsg.Forge // the merged messages Arrive was handed, by stream sequence, until their merge is taken in or dropped
|
|
// Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil
|
|
// takes every merge in at once, as before the take-in step.
|
|
Pending Store
|
|
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
|
|
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
|
|
// ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker,
|
|
// A-248); nil validates nothing, and a revision is taken in as before.
|
|
ValidationAsks ValidationAsks
|
|
// AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an
|
|
// object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate.
|
|
AskVerb buildstep.AskVerb
|
|
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
|
|
Runs Store
|
|
runMu sync.Mutex // one run event at a time (runcandidates.go)
|
|
pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go)
|
|
}
|
|
|
|
func (ix *Indexer) logf(format string, args ...any) {
|
|
if ix.Logf != nil {
|
|
ix.Logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
|
|
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
|
|
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
|
|
// merged message also writes the fingerprint of each module of its repository (modules.go).
|
|
func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error {
|
|
_, err := ix.ApplyMessage(ctx, f)
|
|
return err
|
|
}
|
|
|
|
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
|
|
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
|
|
func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) {
|
|
ix.kickJudge(ctx)
|
|
var applied Applied
|
|
if f.Kind == forgemsg.ForgeOpen {
|
|
return applied, ix.applyOpen(ctx, f)
|
|
}
|
|
if f.Kind != forgemsg.ForgePull && f.Kind != forgemsg.ForgeMerged {
|
|
return applied, nil
|
|
}
|
|
lateMerge := false
|
|
if f.Kind == forgemsg.ForgeMerged && ix.Modules != nil {
|
|
w, err := ix.applyMerged(ctx, f)
|
|
applied = Applied{Written: w.written, Late: w.late}
|
|
if err != nil {
|
|
return applied, err
|
|
}
|
|
ix.logf("%s", w.said(f.Number))
|
|
if w.late == "" { // a late message leaves the taken key, which names a later merge
|
|
if err := ix.writeTakenKnown(ctx, f, w.asked); err != nil {
|
|
return applied, err
|
|
}
|
|
// The control records (ADR 0565 §2): the catalogue's predefined list, and the open pull requests' config
|
|
// declarations judged again against the trunk this merge moved.
|
|
if err := ix.writePredefined(ctx, f); err != nil {
|
|
return applied, err
|
|
}
|
|
if err := ix.configCandidatesAfterMerge(ctx, f, writtenModules(w.written)); err != nil {
|
|
return applied, err
|
|
}
|
|
// Each package's name in its own ecosystem, which the builder numbers it under (design 32), kept before the
|
|
// manifests are judged: a manifest of this merge naming a package this merge brings is judged knowing its name.
|
|
if err := ix.KeepPackageNames(ctx, f); err != nil {
|
|
return applied, err
|
|
}
|
|
// Every manifest judged again: a rule between manifests can turn on the one this merge moved (ADR 0578 §1).
|
|
// What this take-in newly wrote is fresh: judged in full (ADR 0579 §1).
|
|
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
|
|
// Judged again at the next message or source revision heard (kickJudge), as a failed start pass is.
|
|
ix.judgePending.Store(true)
|
|
ix.logf("judge: the manifests are not judged after %s#%d: %v; judged again at the next message heard", f.Repository, f.Number, err)
|
|
}
|
|
}
|
|
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
|
|
lateMerge = w.late != ""
|
|
}
|
|
// The seat definitions of the contracts modules it carries (DLV-I17, I18): a late merge moves no trunk's.
|
|
if !lateMerge {
|
|
if err := ix.ApplySeatDefinitions(ctx, f); err != nil {
|
|
return applied, err
|
|
}
|
|
}
|
|
var digests map[string]string
|
|
var held map[string]meshcatalog.ModulesByModuleValue
|
|
if f.Kind == forgemsg.ForgePull {
|
|
var err error
|
|
if digests, err = ix.keepManifests(ctx, f); err != nil {
|
|
return applied, err
|
|
}
|
|
if held, err = ix.heldModules(ctx); err != nil {
|
|
return applied, err
|
|
}
|
|
// Each manifest the pull carries is judged by this judge, unless it holds a verdict already that this judge made or
|
|
// that accepts it: a refusal an older judge made never stands for a candidate (or a digest back on the trunk).
|
|
over := map[string]string{}
|
|
for _, digest := range digests {
|
|
if ix.Manifests == nil {
|
|
break
|
|
}
|
|
if value, _, found, err := ix.Manifests.Get(ctx, digest); err == nil && found {
|
|
if kept, err := meshcatalogclient.DecodeManifest(value); err == nil {
|
|
over[kept.Module] = digest
|
|
}
|
|
}
|
|
}
|
|
ix.judgeDigests(ctx, over, fmt.Sprintf("%s#%d", f.Repository, f.Number))
|
|
if err := ix.configCandidateOfPull(ctx, f, held); err != nil {
|
|
if !errors.Is(err, ErrUnreadable) {
|
|
return applied, err
|
|
}
|
|
ix.logf("%s#%d: its config declarations are not written: %v", f.Repository, f.Number, err)
|
|
}
|
|
}
|
|
return applied, ix.writeCandidate(ctx, f, digests, held)
|
|
}
|
|
|
|
// removeCandidate takes a merged pull request out of its group's candidate and writes nothing else of its message: for a
|
|
// merge a newer merge of the same repository covers (it needs only the repository, number and branch).
|
|
func (ix *Indexer) removeCandidate(ctx context.Context, f forgemsg.Forge) error {
|
|
return ix.writeCandidate(ctx, f, nil, nil)
|
|
}
|
|
|
|
// writeCandidate writes the group's candidate with the pull request f names (taken out for a merged or closed one).
|
|
func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) error {
|
|
if trunkLike[f.Branch] {
|
|
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
|
|
return nil
|
|
}
|
|
key, err := CandidatesKey(f.Branch)
|
|
if err != nil {
|
|
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
|
}
|
|
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
|
var cur *meshcatalog.CandidatesByGroupValue
|
|
if found {
|
|
c, err := DecodeCandidate(value)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
|
}
|
|
cur = &c
|
|
}
|
|
next := ix.next(cur, f, digests, held)
|
|
if next != nil {
|
|
ix.withPreviews(ctx, next)
|
|
if err := ix.refuseDisagreement(*next); err != nil {
|
|
return nil, false, err
|
|
}
|
|
}
|
|
switch {
|
|
case next == nil && cur == nil, next != nil && cur != nil && SameCandidate(*cur, *next):
|
|
return nil, false, nil
|
|
case next == nil:
|
|
return nil, true, nil
|
|
}
|
|
data, err := EncodeCandidate(*next)
|
|
return data, false, err
|
|
})
|
|
}
|
|
|
|
// applyOpen takes an open message into the candidates (novox/hq ADR 0443 §5): every pull request it does not name is
|
|
// taken out of its group, and a group left with none is deleted. What the message names is left as it is. A pull request
|
|
// opened after the bridge read the forge is heard after this message and added again by its own.
|
|
func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error {
|
|
named := make(map[forgemsg.OpenPull]bool, len(f.Open))
|
|
for _, p := range f.Open {
|
|
named[p] = true
|
|
}
|
|
keys, err := ix.Store.Keys(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("listing the candidates: %w", err)
|
|
}
|
|
for _, key := range keys {
|
|
if meshcatalogclient.IsTryGroup(key) {
|
|
continue // a run's candidate is no pull request's: its run owns it (runcandidates.go)
|
|
}
|
|
err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
|
if !found {
|
|
return nil, false, nil
|
|
}
|
|
c, err := DecodeCandidate(value)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
|
}
|
|
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p meshcatalog.CandidatesByGroupValuePullsItem) bool {
|
|
return !named[forgemsg.OpenPull{Repository: p.Repository, Number: int(p.Number)}]
|
|
})
|
|
switch {
|
|
case len(kept) == len(c.Pulls):
|
|
return nil, false, nil
|
|
case len(kept) == 0:
|
|
ix.logf("%s: no pull request of the group is open, so no candidate", key)
|
|
return nil, true, nil
|
|
}
|
|
heads := make([]meshdeliveryclient.Head, len(kept))
|
|
for i, p := range kept {
|
|
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
|
|
}
|
|
hash, err := meshdeliveryclient.HeadsHash(heads)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("%w: candidates.%s: %v", ErrUnreadable, key, err)
|
|
}
|
|
c.Pulls, c.Heads = kept, hash
|
|
data, err := EncodeCandidate(c)
|
|
return data, false, err
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// write changes one key of a store by compare-and-set: compute is given the value read (found says there is one) and
|
|
// answers the new value, or del to remove the key, or neither for no change. A conflict reads again, up to writeTries.
|
|
// read, when given, is set to the revision of the value compute was last given.
|
|
//
|
|
// **A value whose bytes are the bytes held is not written** (issue 1966, 2026-11-08: a compare of two map types that never
|
|
// matched rewrote every module on every recompute, each write a views event, and a reader of the views took each as a gap
|
|
// and asked its whole first view again): whatever compute decides, a write that changes nothing but the revision is none.
|
|
func write(ctx context.Context, st Store, key string, compute func(value []byte, found bool) (next []byte, del bool, err error), read ...*uint64) error {
|
|
for try := 0; try < writeTries; try++ {
|
|
value, revision, found, err := st.Get(ctx, key)
|
|
if err != nil {
|
|
return fmt.Errorf("reading %s: %w", key, err)
|
|
}
|
|
next, del, err := compute(value, found)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, r := range read {
|
|
*r = revision
|
|
}
|
|
switch {
|
|
case next == nil && !del, unchanged(next, del, value, found):
|
|
return nil
|
|
case del:
|
|
err = st.Delete(ctx, key, revision)
|
|
case !found:
|
|
err = st.Create(ctx, key, next)
|
|
default:
|
|
err = st.Update(ctx, key, next, revision)
|
|
}
|
|
if errors.Is(err, ErrConflict) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("writing %s: %w", key, err)
|
|
}
|
|
return nil
|
|
}
|
|
return fmt.Errorf("%s changed under every one of %d tries", key, writeTries)
|
|
}
|
|
|
|
// unchanged says that compute's answer next is the value held: write writes nothing then (write).
|
|
func unchanged(next []byte, del bool, held []byte, found bool) bool {
|
|
return found && !del && next != nil && bytes.Equal(next, held)
|
|
}
|
|
|
|
// next is the group after the message; nil when the group is left with no pull request.
|
|
//
|
|
// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of
|
|
// its manifest and why it has no fingerprint of the new form (meshcatalogclient.NoneNoPreview), and the candidate its FPSet (issue 1451); a
|
|
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
|
|
// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir).
|
|
func (ix *Indexer) next(cur *meshcatalog.CandidatesByGroupValue, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) *meshcatalog.CandidatesByGroupValue {
|
|
var pulls []meshcatalog.CandidatesByGroupValuePullsItem
|
|
var base int64
|
|
if cur != nil {
|
|
pulls, base = slices.Clone(cur.Pulls), cur.Base
|
|
}
|
|
at := slices.IndexFunc(pulls, func(p meshcatalog.CandidatesByGroupValuePullsItem) bool {
|
|
return p.Repository == f.Repository && p.Number == int64(f.Number)
|
|
})
|
|
if f.Kind == forgemsg.ForgeMerged || f.Action == forgemsg.PullClosed {
|
|
if at < 0 {
|
|
return cur
|
|
}
|
|
pulls = slices.Delete(pulls, at, at+1)
|
|
} else {
|
|
entry := meshcatalog.CandidatesByGroupValuePullsItem{Repository: f.Repository, Number: int64(f.Number), Head: f.Head, Modules: []string{}}
|
|
switch {
|
|
case f.DirsKnown:
|
|
for _, dir := range f.Dirs {
|
|
m := moduleOfDir(held, f, dir)
|
|
if !slices.Contains(entry.Modules, m) {
|
|
entry.Modules = append(entry.Modules, m)
|
|
}
|
|
if digest, ok := digests[dir]; ok {
|
|
if entry.Fps == nil {
|
|
entry.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
|
|
}
|
|
entry.Fps[m] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: digest, None2: meshcatalogclient.NoneNoPreview}
|
|
}
|
|
}
|
|
case at >= 0 && pulls[at].Head == f.Head:
|
|
// Its directories are not known now, for the head the group already holds: what it knew stands.
|
|
entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps
|
|
}
|
|
if at >= 0 {
|
|
pulls[at] = entry
|
|
} else {
|
|
pulls = append(pulls, entry)
|
|
}
|
|
}
|
|
if len(pulls) == 0 {
|
|
return nil
|
|
}
|
|
slices.SortFunc(pulls, func(a, b meshcatalog.CandidatesByGroupValuePullsItem) int {
|
|
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
|
|
return c
|
|
}
|
|
return cmp.Compare(a.Number, b.Number)
|
|
})
|
|
heads := make([]meshdeliveryclient.Head, len(pulls))
|
|
for i, p := range pulls {
|
|
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
|
|
}
|
|
hash, _ := meshdeliveryclient.HeadsHash(heads)
|
|
return &meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: f.Branch, Base: base, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}
|
|
}
|
|
|
|
// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the
|
|
// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
|
|
// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
|
|
// request is judged again.
|
|
func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) error {
|
|
first := map[string]meshcatalog.CandidatesByGroupValuePullsItem{}
|
|
for _, p := range c.Pulls {
|
|
for module, m := range p.Fps {
|
|
before, ok := first[module]
|
|
if !ok {
|
|
first[module] = p
|
|
continue
|
|
}
|
|
if !reflect.DeepEqual(before.Fps[module], m) {
|
|
err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
|
|
ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|