Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/runcandidates.go
T
jschoubben cf93d30150 Names N3 (indexer): the indexer names its own buckets from its credential's module
Its 26 `<Module>_<state>` bucket constants (candidates, manifests, verdicts, packages, taken, the
control records, pins, pending and the rest), its consumers, findings and first views were named
from the constant Module; they are now index.Bucket(cred.Module, <state>), and its words name
the state where they named the bucket.

Builds: design 47 G A-296

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 19:18:56 +01:00

296 lines
12 KiB
Go

package index
// **A run on request's candidate** (design 47 G §64, §190, §321; A-160, A-174): the runner says `run-started` with the run's
// members, each a module at a named commit on a branch; the indexer builds the run's candidate from it, `candidates.run-<id>`,
// and drops it once every member's pipeline of that run has ended (`pipeline-ended` naming the run). Each member maps to its
// repository and directory by its registration, and its manifest and fingerprint come from the same take-in the indexer
// computes for a watched branch (its files read through the seat git), over the revision view of the member's branch when its
// registration watches that branch, else over the trunk's (A-93). No checker preview is heard and no checker tool is asked.
//
// What the indexer knows of each run is kept in its state runs (key run-<id>: the members and the modules whose pipeline
// ended), written on run-started and on each of the run's pipeline-ended and removed with the candidate, so a restart between
// two members' ends loses nothing (A-174). A pipeline-ended heard before its run-started is kept all the same: the run-started
// that follows finds it.
import (
"context"
"encoding/json"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// RunsState is the indexer's state of the runs on request it holds a candidate for (A-174): the bucket
// mesh-delivery-indexer_runs, one key per run, run-<id>.
const (
RunsState = "runs"
)
// Run is one value of the state runs: the run's members as run-started named them, and the modules whose pipeline of the
// run has ended.
type Run struct {
V int `json:"v"`
Run string `json:"run"`
Members []meshpipelinerunner.Member `json:"members,omitempty"`
Ended []string `json:"ended,omitempty"`
At time.Time `json:"at"`
}
// RunGroup is the candidate's group, and the runs key, of a run: run-<id> (A-160).
func RunGroup(run string) (string, error) {
g := "run-" + strings.TrimSpace(run)
if strings.TrimSpace(run) == "" || !meshcatalogclient.IsTryGroup(g) {
return "", fmt.Errorf("%w: %q is no run's id", ErrUnreadable, run)
}
return g, nil
}
// done says every member's pipeline of the run has ended; a run whose run-started is not heard yet is not done.
func (r Run) done() bool {
if len(r.Members) == 0 {
return false
}
for _, m := range r.Members {
if !slices.Contains(r.Ended, m.Module) {
return false
}
}
return true
}
// StartRun takes a run on request in, as the runner said it started: its members are kept in the state runs and its
// candidate is written; a run every member of which has ended already (its ends heard first) is dropped instead.
func (ix *Indexer) StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error {
group, err := RunGroup(e.Run)
if err != nil {
return err
}
if len(e.Members) == 0 {
return fmt.Errorf("%w: the run %s names no member", ErrUnreadable, e.Run)
}
ix.runMu.Lock()
defer ix.runMu.Unlock()
r, err := ix.updateRun(ctx, group, func(r *Run) { r.Members = slices.Clone(e.Members) })
if err != nil {
return err
}
if r.done() {
return ix.dropRun(ctx, group)
}
return ix.writeRunCandidate(ctx, group, r.Members)
}
// EndRunPipeline takes one pipeline-ended in: of a run's pipeline, its module is kept as ended, and once every member's has
// ended the candidate and the run's key are removed. A pipeline of no run is none of this.
func (ix *Indexer) EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error {
if e.Run == "" {
return nil
}
group, err := RunGroup(e.Run)
if err != nil {
return err
}
ix.runMu.Lock()
defer ix.runMu.Unlock()
r, err := ix.updateRun(ctx, group, func(r *Run) {
if e.Module != "" && !slices.Contains(r.Ended, e.Module) {
r.Ended = append(r.Ended, e.Module)
slices.Sort(r.Ended)
}
})
if err != nil {
return err
}
if !r.done() {
return nil
}
return ix.dropRun(ctx, group)
}
// updateRun reads the run's key, changes it and writes it back, made when absent.
func (ix *Indexer) updateRun(ctx context.Context, group string, change func(*Run)) (Run, error) {
if ix.Runs == nil {
return Run{}, fmt.Errorf("the state %s is not bound, so the run %s is not kept", RunsState, group)
}
var out Run
err := write(ctx, ix.Runs, group, func(value []byte, found bool) ([]byte, bool, error) {
r := Run{V: 1, Run: strings.TrimPrefix(group, "run-")}
if found {
if err := json.Unmarshal(value, &r); err != nil {
return nil, false, fmt.Errorf("%s.%s holds what is not a run: %w", RunsState, group, err)
}
}
change(&r)
r.At = ix.now().UTC()
out = r
data, err := json.Marshal(r)
return data, false, err
})
return out, err
}
// dropRun removes the run's candidate, then its key.
func (ix *Indexer) dropRun(ctx context.Context, group string) error {
if ix.Store != nil {
key, err := meshcatalogclient.CandidatesKey(group)
if err != nil {
return err
}
if err := write(ctx, ix.Store, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
return fmt.Errorf("the candidate of %s is not dropped: %w", group, err)
}
}
if err := write(ctx, ix.Runs, group, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
return fmt.Errorf("the run %s is not removed: %w", group, err)
}
ix.logf("the run %s: every member's pipeline ended, its candidate is dropped", group)
return nil
}
// RunBase is the branch whose revision view a member's take-in starts from (A-174): the member's own branch when its
// registration watches it, else the trunk.
func RunBase(reg meshcatalog.Registration, branch string) string {
if branch != "" && slices.Contains(reg.Branches, branch) {
return branch
}
return TrunkBranch
}
// runMember is one member placed: its registration's repository and directory, and the branch its take-in starts from.
type runMember struct {
meshpipelinerunner.Member
repository, dir, base string
}
// writeRunCandidate writes the run's candidate: one pull of number 0 per member, its module's manifest and fingerprint as
// the take-in of its commit over its base's revision view computes them, its manifests judged with the seat definitions
// the members carry, as a revision's are.
func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members []meshpipelinerunner.Member) error {
if ix.Store == nil || ix.Read == nil {
return nil
}
if ix.Registry == nil || ix.Registry.Registrations == nil || ix.Revisions == nil {
return fmt.Errorf("the run %s: the indexer holds no registrations or no revisions, so its members cannot be taken in", group)
}
var placed []runMember
for _, m := range members {
reg, found, err := ix.Registry.Registration(ctx, m.Module)
if err != nil {
return fmt.Errorf("the run %s: the registration of %s: %w", group, m.Module, err)
}
if !found {
return fmt.Errorf("%w: the run %s: %s has no registration, so its repository is not known", ErrUnreadable, group, m.Module)
}
dir := strings.Trim(reg.Path, "/")
if dir == meshcatalogclient.RootPath {
dir = ""
}
placed = append(placed, runMember{Member: m, repository: trunk.RepositoryName(reg.Origin), dir: dir, base: RunBase(reg, m.Branch)})
}
slices.SortFunc(placed, func(a, b runMember) int {
return strings.Compare(a.repository+"\x00"+a.Module, b.repository+"\x00"+b.Module)
})
heads := make([]meshdeliveryclient.Head, len(placed))
for i, m := range placed {
heads[i] = meshdeliveryclient.Head{Repository: m.repository, Head: m.Commit}
}
hash, err := meshdeliveryclient.HeadsHash(heads)
if err != nil {
return err
}
pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed))
var entries []PendingEntry
over := map[string]string{}
// One take-in per base branch: the members on one base are taken in together, so one member sees another's.
bases := map[string]*dryRun{}
for i, m := range placed {
d, ok := bases[m.base]
if !ok {
view, nodes, _, err := BranchView(ctx, ix.Revisions, m.base)
if err != nil {
return fmt.Errorf("the run %s: the revision view of %s: %w", group, m.base, err)
}
d = ix.dry()
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
d.ix.Logf = func(format string, args ...any) {
ix.logf("the run "+group+", computed and not written: "+format, args...)
}
bases[m.base] = d
}
read, err := ix.Read(ctx, m.repository, m.Commit, m.dir)
if err != nil {
return fmt.Errorf("the run %s: reading %s of %s at %.12s: %w", group, m.Module, m.repository, m.Commit, err)
}
raw := read.Manifests[m.dir]
if raw == "" {
return fmt.Errorf("%w: the run %s: %s at %.12s holds no manifest at %q", ErrUnreadable, group, m.repository, m.Commit, m.dir)
}
if name := manifestModule(raw); name != m.Module {
return fmt.Errorf("%w: the run %s: the manifest at %q of %s at %.12s names %q, not %s", ErrUnreadable, group, m.dir, m.repository, m.Commit, name, m.Module)
}
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: m.repository, Merge: m.Commit, Root: read.Root,
Trees: map[string]string{m.dir: read.Trees[m.dir]}, Manifests: map[string]string{m.dir: raw}}
if l := read.Locks[m.dir]; len(l) > 0 {
f.Locks = map[string]map[string]string{m.dir: l}
}
if p := read.Packages[m.dir]; len(p) > 0 {
f.Packages = map[string][]string{m.dir: p}
}
if _, err := d.ix.applyManifests(ctx, f, false); err != nil {
return fmt.Errorf("the run %s: taking %s in: %w", group, m.Module, err)
}
key, err := meshcatalogclient.ModulesKey(m.Module)
if err != nil {
return err
}
value, _, found, err := d.ix.Modules.Get(ctx, key)
if err != nil {
return err
}
if !found {
return fmt.Errorf("the run %s: the take-in of %s at %.12s wrote no module (the journal says why)", group, m.Module, m.Commit)
}
mv, err := meshcatalogclient.DecodeModule(value)
if err != nil {
return err
}
pulls[i] = meshcatalog.CandidatesByGroupValuePullsItem{Repository: m.repository, Head: m.Commit, Modules: []string{m.Module},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{m.Module: {Manifest: mv.Manifest, Fp2: mv.Fp2, None2: mv.None2,
Packages: mv.Packages}}}
over[m.Module] = mv.Manifest
entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.repository, Merge: m.Commit, Manifests: map[string]string{m.dir: raw}}})
}
seats, err := ix.revisionSeats(ctx, entries)
if err != nil {
return fmt.Errorf("the run %s: reading the seat definitions of its members: %w", group, err)
}
ix.judgeDigestsWith(ctx, seats, over, "the run "+group)
c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls}
c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls)
if err := ix.refuseDisagreement(c); err != nil {
return err
}
data, err := EncodeCandidate(c)
if err != nil {
return err
}
key, err := meshcatalogclient.CandidatesKey(group)
if err != nil {
return err
}
if err := write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil }); err != nil {
return err
}
ix.logf("the run %s: its candidate of %d members is written", group, len(placed))
return nil
}