Its 26 `<Module>_<state>` bucket constants (candidates, manifests, verdicts, packages, taken, the control records, pins, pending and the rest), its consumers, findings and first views were named from the constant Module; they are now index.Bucket(cred.Module, <state>), and its words name the state where they named the bucket. Builds: design 47 G A-296 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
296 lines
12 KiB
Go
296 lines
12 KiB
Go
package index
|
|
|
|
// **A run on request's candidate** (design 47 G §64, §190, §321; A-160, A-174): the runner says `run-started` with the run's
|
|
// members, each a module at a named commit on a branch; the indexer builds the run's candidate from it, `candidates.run-<id>`,
|
|
// and drops it once every member's pipeline of that run has ended (`pipeline-ended` naming the run). Each member maps to its
|
|
// repository and directory by its registration, and its manifest and fingerprint come from the same take-in the indexer
|
|
// computes for a watched branch (its files read through the seat git), over the revision view of the member's branch when its
|
|
// registration watches that branch, else over the trunk's (A-93). No checker preview is heard and no checker tool is asked.
|
|
//
|
|
// What the indexer knows of each run is kept in its state runs (key run-<id>: the members and the modules whose pipeline
|
|
// ended), written on run-started and on each of the run's pipeline-ended and removed with the candidate, so a restart between
|
|
// two members' ends loses nothing (A-174). A pipeline-ended heard before its run-started is kept all the same: the run-started
|
|
// that follows finds it.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// RunsState is the indexer's state of the runs on request it holds a candidate for (A-174): the bucket
|
|
// mesh-delivery-indexer_runs, one key per run, run-<id>.
|
|
const (
|
|
RunsState = "runs"
|
|
)
|
|
|
|
// Run is one value of the state runs: the run's members as run-started named them, and the modules whose pipeline of the
|
|
// run has ended.
|
|
type Run struct {
|
|
V int `json:"v"`
|
|
Run string `json:"run"`
|
|
Members []meshpipelinerunner.Member `json:"members,omitempty"`
|
|
Ended []string `json:"ended,omitempty"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
|
|
// RunGroup is the candidate's group, and the runs key, of a run: run-<id> (A-160).
|
|
func RunGroup(run string) (string, error) {
|
|
g := "run-" + strings.TrimSpace(run)
|
|
if strings.TrimSpace(run) == "" || !meshcatalogclient.IsTryGroup(g) {
|
|
return "", fmt.Errorf("%w: %q is no run's id", ErrUnreadable, run)
|
|
}
|
|
return g, nil
|
|
}
|
|
|
|
// done says every member's pipeline of the run has ended; a run whose run-started is not heard yet is not done.
|
|
func (r Run) done() bool {
|
|
if len(r.Members) == 0 {
|
|
return false
|
|
}
|
|
for _, m := range r.Members {
|
|
if !slices.Contains(r.Ended, m.Module) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// StartRun takes a run on request in, as the runner said it started: its members are kept in the state runs and its
|
|
// candidate is written; a run every member of which has ended already (its ends heard first) is dropped instead.
|
|
func (ix *Indexer) StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error {
|
|
group, err := RunGroup(e.Run)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(e.Members) == 0 {
|
|
return fmt.Errorf("%w: the run %s names no member", ErrUnreadable, e.Run)
|
|
}
|
|
ix.runMu.Lock()
|
|
defer ix.runMu.Unlock()
|
|
r, err := ix.updateRun(ctx, group, func(r *Run) { r.Members = slices.Clone(e.Members) })
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if r.done() {
|
|
return ix.dropRun(ctx, group)
|
|
}
|
|
return ix.writeRunCandidate(ctx, group, r.Members)
|
|
}
|
|
|
|
// EndRunPipeline takes one pipeline-ended in: of a run's pipeline, its module is kept as ended, and once every member's has
|
|
// ended the candidate and the run's key are removed. A pipeline of no run is none of this.
|
|
func (ix *Indexer) EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error {
|
|
if e.Run == "" {
|
|
return nil
|
|
}
|
|
group, err := RunGroup(e.Run)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ix.runMu.Lock()
|
|
defer ix.runMu.Unlock()
|
|
r, err := ix.updateRun(ctx, group, func(r *Run) {
|
|
if e.Module != "" && !slices.Contains(r.Ended, e.Module) {
|
|
r.Ended = append(r.Ended, e.Module)
|
|
slices.Sort(r.Ended)
|
|
}
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !r.done() {
|
|
return nil
|
|
}
|
|
return ix.dropRun(ctx, group)
|
|
}
|
|
|
|
// updateRun reads the run's key, changes it and writes it back, made when absent.
|
|
func (ix *Indexer) updateRun(ctx context.Context, group string, change func(*Run)) (Run, error) {
|
|
if ix.Runs == nil {
|
|
return Run{}, fmt.Errorf("the state %s is not bound, so the run %s is not kept", RunsState, group)
|
|
}
|
|
var out Run
|
|
err := write(ctx, ix.Runs, group, func(value []byte, found bool) ([]byte, bool, error) {
|
|
r := Run{V: 1, Run: strings.TrimPrefix(group, "run-")}
|
|
if found {
|
|
if err := json.Unmarshal(value, &r); err != nil {
|
|
return nil, false, fmt.Errorf("%s.%s holds what is not a run: %w", RunsState, group, err)
|
|
}
|
|
}
|
|
change(&r)
|
|
r.At = ix.now().UTC()
|
|
out = r
|
|
data, err := json.Marshal(r)
|
|
return data, false, err
|
|
})
|
|
return out, err
|
|
}
|
|
|
|
// dropRun removes the run's candidate, then its key.
|
|
func (ix *Indexer) dropRun(ctx context.Context, group string) error {
|
|
if ix.Store != nil {
|
|
key, err := meshcatalogclient.CandidatesKey(group)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := write(ctx, ix.Store, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
|
|
return fmt.Errorf("the candidate of %s is not dropped: %w", group, err)
|
|
}
|
|
}
|
|
if err := write(ctx, ix.Runs, group, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
|
|
return fmt.Errorf("the run %s is not removed: %w", group, err)
|
|
}
|
|
ix.logf("the run %s: every member's pipeline ended, its candidate is dropped", group)
|
|
return nil
|
|
}
|
|
|
|
// RunBase is the branch whose revision view a member's take-in starts from (A-174): the member's own branch when its
|
|
// registration watches it, else the trunk.
|
|
func RunBase(reg meshcatalog.Registration, branch string) string {
|
|
if branch != "" && slices.Contains(reg.Branches, branch) {
|
|
return branch
|
|
}
|
|
return TrunkBranch
|
|
}
|
|
|
|
// runMember is one member placed: its registration's repository and directory, and the branch its take-in starts from.
|
|
type runMember struct {
|
|
meshpipelinerunner.Member
|
|
repository, dir, base string
|
|
}
|
|
|
|
// writeRunCandidate writes the run's candidate: one pull of number 0 per member, its module's manifest and fingerprint as
|
|
// the take-in of its commit over its base's revision view computes them, its manifests judged with the seat definitions
|
|
// the members carry, as a revision's are.
|
|
func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members []meshpipelinerunner.Member) error {
|
|
if ix.Store == nil || ix.Read == nil {
|
|
return nil
|
|
}
|
|
if ix.Registry == nil || ix.Registry.Registrations == nil || ix.Revisions == nil {
|
|
return fmt.Errorf("the run %s: the indexer holds no registrations or no revisions, so its members cannot be taken in", group)
|
|
}
|
|
var placed []runMember
|
|
for _, m := range members {
|
|
reg, found, err := ix.Registry.Registration(ctx, m.Module)
|
|
if err != nil {
|
|
return fmt.Errorf("the run %s: the registration of %s: %w", group, m.Module, err)
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("%w: the run %s: %s has no registration, so its repository is not known", ErrUnreadable, group, m.Module)
|
|
}
|
|
dir := strings.Trim(reg.Path, "/")
|
|
if dir == meshcatalogclient.RootPath {
|
|
dir = ""
|
|
}
|
|
placed = append(placed, runMember{Member: m, repository: trunk.RepositoryName(reg.Origin), dir: dir, base: RunBase(reg, m.Branch)})
|
|
}
|
|
slices.SortFunc(placed, func(a, b runMember) int {
|
|
return strings.Compare(a.repository+"\x00"+a.Module, b.repository+"\x00"+b.Module)
|
|
})
|
|
heads := make([]meshdeliveryclient.Head, len(placed))
|
|
for i, m := range placed {
|
|
heads[i] = meshdeliveryclient.Head{Repository: m.repository, Head: m.Commit}
|
|
}
|
|
hash, err := meshdeliveryclient.HeadsHash(heads)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed))
|
|
var entries []PendingEntry
|
|
over := map[string]string{}
|
|
// One take-in per base branch: the members on one base are taken in together, so one member sees another's.
|
|
bases := map[string]*dryRun{}
|
|
for i, m := range placed {
|
|
d, ok := bases[m.base]
|
|
if !ok {
|
|
view, nodes, _, err := BranchView(ctx, ix.Revisions, m.base)
|
|
if err != nil {
|
|
return fmt.Errorf("the run %s: the revision view of %s: %w", group, m.base, err)
|
|
}
|
|
d = ix.dry()
|
|
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
|
|
d.ix.Logf = func(format string, args ...any) {
|
|
ix.logf("the run "+group+", computed and not written: "+format, args...)
|
|
}
|
|
bases[m.base] = d
|
|
}
|
|
read, err := ix.Read(ctx, m.repository, m.Commit, m.dir)
|
|
if err != nil {
|
|
return fmt.Errorf("the run %s: reading %s of %s at %.12s: %w", group, m.Module, m.repository, m.Commit, err)
|
|
}
|
|
raw := read.Manifests[m.dir]
|
|
if raw == "" {
|
|
return fmt.Errorf("%w: the run %s: %s at %.12s holds no manifest at %q", ErrUnreadable, group, m.repository, m.Commit, m.dir)
|
|
}
|
|
if name := manifestModule(raw); name != m.Module {
|
|
return fmt.Errorf("%w: the run %s: the manifest at %q of %s at %.12s names %q, not %s", ErrUnreadable, group, m.dir, m.repository, m.Commit, name, m.Module)
|
|
}
|
|
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: m.repository, Merge: m.Commit, Root: read.Root,
|
|
Trees: map[string]string{m.dir: read.Trees[m.dir]}, Manifests: map[string]string{m.dir: raw}}
|
|
if l := read.Locks[m.dir]; len(l) > 0 {
|
|
f.Locks = map[string]map[string]string{m.dir: l}
|
|
}
|
|
if p := read.Packages[m.dir]; len(p) > 0 {
|
|
f.Packages = map[string][]string{m.dir: p}
|
|
}
|
|
if _, err := d.ix.applyManifests(ctx, f, false); err != nil {
|
|
return fmt.Errorf("the run %s: taking %s in: %w", group, m.Module, err)
|
|
}
|
|
key, err := meshcatalogclient.ModulesKey(m.Module)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
value, _, found, err := d.ix.Modules.Get(ctx, key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
return fmt.Errorf("the run %s: the take-in of %s at %.12s wrote no module (the journal says why)", group, m.Module, m.Commit)
|
|
}
|
|
mv, err := meshcatalogclient.DecodeModule(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pulls[i] = meshcatalog.CandidatesByGroupValuePullsItem{Repository: m.repository, Head: m.Commit, Modules: []string{m.Module},
|
|
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{m.Module: {Manifest: mv.Manifest, Fp2: mv.Fp2, None2: mv.None2,
|
|
Packages: mv.Packages}}}
|
|
over[m.Module] = mv.Manifest
|
|
entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.repository, Merge: m.Commit, Manifests: map[string]string{m.dir: raw}}})
|
|
}
|
|
seats, err := ix.revisionSeats(ctx, entries)
|
|
if err != nil {
|
|
return fmt.Errorf("the run %s: reading the seat definitions of its members: %w", group, err)
|
|
}
|
|
ix.judgeDigestsWith(ctx, seats, over, "the run "+group)
|
|
c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls}
|
|
c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls)
|
|
if err := ix.refuseDisagreement(c); err != nil {
|
|
return err
|
|
}
|
|
data, err := EncodeCandidate(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
key, err := meshcatalogclient.CandidatesKey(group)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil }); err != nil {
|
|
return err
|
|
}
|
|
ix.logf("the run %s: its candidate of %d members is written", group, len(placed))
|
|
return nil
|
|
}
|