Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/seed_steps.go
T
jschoubben 00b452e57b The readers of the newest trunk revision read the newest again when it is removed under them
ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, the first fill of the versions, the outdated tool's trunk read and the verb
revision without a kept view use readNewestOf; none of them holds keepMu.

Builds: design 47 G A-487d (hq 61bf1215)
Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-19 12:48:14 +01:00

310 lines
12 KiB
Go

package index
import (
"context"
"errors"
"fmt"
"maps"
"path"
"slices"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// StepsVerb is the verb of each build seat the seed asks (design 47 G §704).
const StepsVerb = "steps"
// StepsCaller asks the build seat seat its verb steps (a kept call, ADR 0668 §1): the answer, or why none came.
type StepsCaller func(ctx context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error)
// ModuleManifest is the file name of a module's manifest in its directory.
const ModuleManifest = "module.json"
// errNoBuildSeat says no build seat's acting rule build matches an artifact's language or kind.
var errNoBuildSeat = errors.New("no seat builds the artifact")
// buildSeatOf is the seat whose acting rule `build` has `on` matching the artifact's language or kind (design 47 G §704,
// §203, §324), read from the seat list defs as BuildModulesOf reads it; none, or more than one, is said.
func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (string, error) {
var seats []string
for _, name := range slices.Sorted(maps.Keys(defs)) {
for _, r := range defs[name].Acts {
if r.Step != seatdef.StepBuild {
continue
}
if r.ActsOn(a.Language) || (r.On == seatdef.OnKindImage && a.Kind == "image") {
seats = append(seats, name)
break
}
}
}
switch len(seats) {
case 0:
return "", fmt.Errorf("%w %s (language %q, kind %q)", errNoBuildSeat, a.Ref, a.Language, a.Kind)
case 1:
return seats[0], nil
}
return "", fmt.Errorf("the seats %v each build the artifact %s", seats, a.Ref)
}
// SeedModulesFrom reads the modules the newest trunk revision holds for the seed (design 47 G §704, §8): for each artifact of
// each module the seat list's build seat answers steps with {module, artifact, fingerprint, fetch} from the module's trunk
// entry, the fetch its repository, commit, directory and the artifact's named paths. A seat that does not answer leaves
// its artifact unanswered, which holds the seed; its call-unanswered finding (scope call, id <seat>.steps) is written once
// it is unanswered five minutes and cleared when it answers.
func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.Definition, steps StepsCaller) ([]SeedModule, error) {
if ix.Revisions == nil {
return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from")
}
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first")
}
seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made
unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind
for _, e := range entries {
for _, a := range e.Artifacts {
if sdkstep.PackedOrFetched(a.Language, a.Kind) {
continue
}
seat, err := buildSeatOf(defs, a)
switch {
case errors.Is(err, errNoBuildSeat):
unbuilt[e.Module] = append(unbuilt[e.Module], unbuiltWords(a))
continue
case err != nil:
return nil, fmt.Errorf("%s: %w", e.Module, err)
}
seatOf[a.Ref] = seat
}
}
answered := map[string]bool{}
unanswered := map[string]error{}
var out []SeedModule
for _, e := range entries {
m := SeedModule{Module: e.Module, Origin: e.Repository, Path: path.Join(e.Path, ModuleManifest)}
if words, ok := unbuilt[e.Module]; ok {
m.LeftOut = "no build seat builds " + strings.Join(words, ", ")
out = append(out, m)
continue
}
for _, a := range e.Artifacts {
name := artifactName(a.Ref)
if sdkstep.PackedOrFetched(a.Language, a.Kind) {
// The builder packs or fetches it: no steps call, and its lint, unit and integration are bypass (A-324).
m.Artifacts = append(m.Artifacts, SeedArtifact{Name: name, Answered: true})
continue
}
seat := seatOf[a.Ref]
sa := SeedArtifact{Name: name, Seat: seat}
in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp,
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: maps.Clone(e.Packages)}}
// The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause
// 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest).
if paths := namedPaths(e, name); len(paths) > 0 {
in.Fetch.Dir, in.Fetch.Paths = e.Path, paths
}
if _, gone := unanswered[seat]; gone {
// the seat did not answer earlier in this seed: its later artifacts are not asked again (A-392)
m.Artifacts = append(m.Artifacts, sa)
continue
}
ans, err := steps(ctx, seat, in)
switch {
case errors.Is(err, ErrAskClosed): // the entry is no longer held: not said unanswered (A-190)
case err != nil:
unanswered[seat] = err
ix.logf("seed: the seat %s did not answer steps: %v; its later artifacts are not asked in this seed", seat, err)
case ans.Error != "":
unanswered[seat] = fmt.Errorf("%s", ans.Error)
ix.logf("seed: the seat %s did not answer steps: %s; its later artifacts are not asked in this seed", seat, ans.Error)
default:
sa.Answered = true
answered[seat] = true
for _, s := range ans.Steps {
sa.Steps = append(sa.Steps, s.Step)
}
}
m.Artifacts = append(m.Artifacts, sa)
}
out = append(out, m)
}
ix.stepsFindings(ctx, answered, unanswered)
ix.leftOutFindings(ctx, entries, out)
return out, nil
}
// unbuiltWords names an artifact no build seat builds, with its language, else its kind.
func unbuiltWords(a meshcatalog.Artifact) string {
if a.Language != "" {
return fmt.Sprintf("%s (language %s)", a.Ref, a.Language)
}
return fmt.Sprintf("%s (kind %q)", a.Ref, a.Kind)
}
// leftOutFindings writes seed-left-out for each module the seed leaves out because no build seat builds an artifact of it,
// and clears it only for a module of which a build seat now builds every artifact, or which left the catalogue; a seed
// applied without the module clears nothing (design 47 G A-324, A-329).
func (ix *Indexer) leftOutFindings(ctx context.Context, entries []meshcatalog.CatalogEntry, mods []SeedModule) {
if ix.Findings == nil {
return
}
left := map[string]bool{}
for _, m := range mods {
if m.LeftOut == "" {
continue
}
left[keyID(m.Module)] = true
f := ix.observed(mcc.ScopeModule, m.Module, SeedLeftOutKind,
fmt.Sprintf("the seed leaves out %s: %s", m.Module, m.LeftOut),
"A module is left out of the first registrations",
fmt.Sprintf("The seed registers no module with an artifact no build seat builds, since a pipeline of it could not run that artifact's steps; %s is left out: %s.", m.Module, m.LeftOut),
"indexer."+SeedLeftOutKind)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("seed: the finding %s of %s is not written: %v", SeedLeftOutKind, m.Module, err)
}
}
inCatalogue := map[string]bool{}
for _, e := range entries {
inCatalogue[keyID(e.Module)] = true
}
for _, f := range ix.Findings.Held() {
if f.Kind != SeedLeftOutKind || f.Scope != mcc.ScopeModule || left[f.ID] {
continue
}
why := "a build seat builds every artifact of it"
if !inCatalogue[f.ID] {
why = "the module left the catalogue"
}
if err := ix.Findings.Clear(ctx, f.Scope, f.ID, f.Kind, why); err != nil {
ix.logf("seed: the finding %s of %s is not cleared: %v", SeedLeftOutKind, f.ID, err)
}
}
}
// artifactName is the artifact's name in an artifact reference <module>:<artifact>.
func artifactName(ref string) string {
for i := len(ref) - 1; i >= 0; i-- {
if ref[i] == ':' {
return ref[i+1:]
}
}
return ref
}
// stepsFindings writes call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, unanswered map[string]error) {
if ix.Findings == nil {
return
}
now := ix.now()
var raise []string
since := map[string]time.Time{}
for seat := range unanswered {
since[seat] = ix.stepsUnansweredSince(ctx, seat, now)
if now.Sub(since[seat]) >= mcc.CallUnansweredAfter {
raise = append(raise, seat)
}
}
for seat := range answered {
if _, bad := unanswered[seat]; !bad {
ix.stepsAnswered(ctx, seat)
}
}
for _, seat := range slices.Sorted(slices.Values(raise)) {
id := seat + "." + StepsVerb
f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered,
fmt.Sprintf("the call %s is unanswered since %s", id, since[seat].UTC().Format(time.RFC3339)),
"A call whose answer is kept is not answered",
fmt.Sprintf("The seed asks %s of every artifact its build seat builds; it is not answered since %s (%v), so the first registrations wait for it.", id, since[seat].UTC().Format(time.RFC3339), unanswered[seat]),
"indexer."+mcc.KindCallUnanswered)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("seed: the finding %s is not written: %v", id, err)
}
}
for seat := range answered {
if _, bad := unanswered[seat]; bad {
continue
}
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, mcc.KindCallUnanswered, "the seat answers steps"); err != nil {
ix.logf("seed: the finding %s.%s is not cleared: %v", seat, StepsVerb, err)
}
}
}
// namedPaths are the named paths the entry keeps for the artifact, by its name or its reference.
func namedPaths(e meshcatalog.CatalogEntry, name string) []string {
if e.Inputs == nil {
return nil
}
if p, ok := e.Inputs.Paths[name]; ok {
return p
}
return e.Inputs.Paths[ArtifactRefOf(e.Module, name)]
}
// StepsSincePrefix opens the keys of the state heard (Heard, A-80) that keep, by build seat, since when its steps
// has not answered (design 47 G A-66): a restart keeps the time, and the finding is written at the next seed attempt past
// five minutes.
const StepsSincePrefix = "steps-since."
// stepsUnansweredSince is the time the seat first failed to answer.
func (ix *Indexer) stepsUnansweredSince(ctx context.Context, seat string, now time.Time) time.Time {
return ix.unansweredSince(ctx, StepsSincePrefix+keys.EscapeName(seat), now)
}
// stepsAnswered forgets the time a seat that answers was unanswered since.
func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) {
ix.answered(ctx, StepsSincePrefix+keys.EscapeName(seat))
}
// unansweredSince is the time the call kept at key first failed to answer, kept in the indexer's state heard (the first
// failure writes now); where no state is kept it is held in memory.
func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Time) time.Time {
if ix.Heard == nil {
ix.mu.Lock()
defer ix.mu.Unlock()
if ix.stepsSince == nil {
ix.stepsSince = map[string]time.Time{}
}
if _, ok := ix.stepsSince[key]; !ok {
ix.stepsSince[key] = now
}
return ix.stepsSince[key]
}
if v, _, found, err := ix.Heard.Get(ctx, key); err == nil && found {
if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil {
return t
}
}
if err := ix.Heard.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil {
ix.logf("the time since %s is unanswered is not kept: %v", key, err)
}
return now
}
// answered forgets the time the call kept at key was unanswered since.
func (ix *Indexer) answered(ctx context.Context, key string) {
if ix.Heard == nil {
ix.mu.Lock()
delete(ix.stepsSince, key)
ix.mu.Unlock()
return
}
if _, rev, found, err := ix.Heard.Get(ctx, key); err == nil && found {
if err := ix.Heard.Delete(ctx, key, rev); err != nil {
ix.logf("the time since %s is unanswered is not forgotten: %v", key, err)
}
}
}