Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/seed_steps.go
T
jschoubben 8af822e3fd SDK (A-324, A-325): one PackedOrFetched for the seed and the runner
The rule of an artifact the builder packs or fetches itself was written twice,
in the indexer's seed and in the runner's placing; both now read the one in
mesh-delivery-sdk's buildstep, the contract of a build step every delivery
part reads.

Builds: design 47 G A-324, A-325

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-13 02:33:38 +01:00

302 lines
11 KiB
Go

package index
import (
"context"
"errors"
"fmt"
"maps"
"path"
"slices"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// StepsVerb is the verb of each build seat the seed asks (design 47 G §704).
const StepsVerb = "steps"
// StepsCaller asks the build seat seat its verb steps (a kept call, ADR 0668 §1): the answer, or why none came.
type StepsCaller func(ctx context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error)
// ModuleManifest is the file name of a module's manifest in its directory.
const ModuleManifest = "module.json"
// errNoBuildSeat says no build seat's acting rule build matches an artifact's language or kind.
var errNoBuildSeat = errors.New("no seat builds the artifact")
// buildSeatOf is the seat whose acting rule `build` has `on` matching the artifact's language or kind (design 47 G §704,
// §203, §324), read from the seat list defs as BuildModulesOf reads it; none, or more than one, is said.
func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (string, error) {
var seats []string
for _, name := range slices.Sorted(maps.Keys(defs)) {
for _, r := range defs[name].Acts {
if r.Step != seatdef.StepBuild {
continue
}
if r.ActsOn(a.Language) || (r.On == seatdef.OnKindImage && a.Kind == "image") {
seats = append(seats, name)
break
}
}
}
switch len(seats) {
case 0:
return "", fmt.Errorf("%w %s (language %q, kind %q)", errNoBuildSeat, a.Ref, a.Language, a.Kind)
case 1:
return seats[0], nil
}
return "", fmt.Errorf("the seats %v each build the artifact %s", seats, a.Ref)
}
// SeedModulesFrom reads the modules the newest trunk revision holds for the seed (design 47 G §704, §8): for each artifact of
// each module the seat list's build seat answers steps with {module, artifact, fingerprint, fetch} from the module's trunk
// entry, the fetch its repository, commit, directory and the artifact's named paths. A seat that does not answer leaves
// its artifact unanswered, which holds the seed; its call-unanswered finding (scope call, id <seat>.steps) is written once
// it is unanswered five minutes and cleared when it answers.
func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.Definition, steps StepsCaller) ([]SeedModule, error) {
if ix.Revisions == nil {
return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from")
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first")
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return nil, err
}
seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made
unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind
for _, e := range entries {
for _, a := range e.Artifacts {
if sdkstep.PackedOrFetched(a.Language, a.Kind) {
continue
}
seat, err := buildSeatOf(defs, a)
switch {
case errors.Is(err, errNoBuildSeat):
unbuilt[e.Module] = append(unbuilt[e.Module], unbuiltWords(a))
continue
case err != nil:
return nil, fmt.Errorf("%s: %w", e.Module, err)
}
seatOf[a.Ref] = seat
}
}
answered := map[string]bool{}
unanswered := map[string]error{}
var out []SeedModule
for _, e := range entries {
m := SeedModule{Module: e.Module, Origin: e.Repository, Path: path.Join(e.Path, ModuleManifest)}
if words, ok := unbuilt[e.Module]; ok {
m.LeftOut = "no build seat builds " + strings.Join(words, ", ")
out = append(out, m)
continue
}
for _, a := range e.Artifacts {
name := artifactName(a.Ref)
if sdkstep.PackedOrFetched(a.Language, a.Kind) {
// The builder packs or fetches it: no steps call, and its lint, unit and integration are bypass (A-324).
m.Artifacts = append(m.Artifacts, SeedArtifact{Name: name, Answered: true})
continue
}
seat := seatOf[a.Ref]
sa := SeedArtifact{Name: name, Seat: seat}
in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp,
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Dir: e.Path, Paths: namedPaths(e, name), Packages: maps.Clone(e.Packages)}}
ans, err := steps(ctx, seat, in)
switch {
case errors.Is(err, ErrAskClosed): // the entry is no longer held: not said unanswered (A-190)
case err != nil:
unanswered[seat] = err
case ans.Error != "":
unanswered[seat] = fmt.Errorf("%s", ans.Error)
default:
sa.Answered = true
answered[seat] = true
for _, s := range ans.Steps {
sa.Steps = append(sa.Steps, s.Step)
}
}
m.Artifacts = append(m.Artifacts, sa)
}
out = append(out, m)
}
ix.stepsFindings(ctx, answered, unanswered)
ix.leftOutFindings(ctx, entries, out)
return out, nil
}
// unbuiltWords names an artifact no build seat builds, with its language, else its kind.
func unbuiltWords(a meshcatalog.Artifact) string {
if a.Language != "" {
return fmt.Sprintf("%s (language %s)", a.Ref, a.Language)
}
return fmt.Sprintf("%s (kind %q)", a.Ref, a.Kind)
}
// leftOutFindings writes seed-left-out for each module the seed leaves out because no build seat builds an artifact of it,
// and clears it only for a module of which a build seat now builds every artifact, or which left the catalogue; a seed
// applied without the module clears nothing (design 47 G A-324, A-329).
func (ix *Indexer) leftOutFindings(ctx context.Context, entries []meshcatalog.CatalogEntry, mods []SeedModule) {
if ix.Findings == nil {
return
}
left := map[string]bool{}
for _, m := range mods {
if m.LeftOut == "" {
continue
}
left[keyID(m.Module)] = true
f := ix.observed(mcc.ScopeModule, m.Module, SeedLeftOutKind,
fmt.Sprintf("the seed leaves out %s: %s", m.Module, m.LeftOut),
"A module is left out of the first registrations",
fmt.Sprintf("The seed registers no module with an artifact no build seat builds, since a pipeline of it could not run that artifact's steps; %s is left out: %s.", m.Module, m.LeftOut),
"indexer."+SeedLeftOutKind)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("seed: the finding %s of %s is not written: %v", SeedLeftOutKind, m.Module, err)
}
}
inCatalogue := map[string]bool{}
for _, e := range entries {
inCatalogue[keyID(e.Module)] = true
}
for _, f := range ix.Findings.Held() {
if f.Kind != SeedLeftOutKind || f.Scope != mcc.ScopeModule || left[f.ID] {
continue
}
why := "a build seat builds every artifact of it"
if !inCatalogue[f.ID] {
why = "the module left the catalogue"
}
if err := ix.Findings.Clear(ctx, f.Scope, f.ID, f.Kind, why); err != nil {
ix.logf("seed: the finding %s of %s is not cleared: %v", SeedLeftOutKind, f.ID, err)
}
}
}
// artifactName is the artifact's name in an artifact reference <module>:<artifact>.
func artifactName(ref string) string {
for i := len(ref) - 1; i >= 0; i-- {
if ref[i] == ':' {
return ref[i+1:]
}
}
return ref
}
// stepsFindings writes call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, unanswered map[string]error) {
if ix.Findings == nil {
return
}
now := ix.now()
var raise []string
since := map[string]time.Time{}
for seat := range unanswered {
since[seat] = ix.stepsUnansweredSince(ctx, seat, now)
if now.Sub(since[seat]) >= mcc.CallUnansweredAfter {
raise = append(raise, seat)
}
}
for seat := range answered {
if _, bad := unanswered[seat]; !bad {
ix.stepsAnswered(ctx, seat)
}
}
for _, seat := range slices.Sorted(slices.Values(raise)) {
id := seat + "." + StepsVerb
f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered,
fmt.Sprintf("the call %s is unanswered since %s", id, since[seat].UTC().Format(time.RFC3339)),
"A call whose answer is kept is not answered",
fmt.Sprintf("The seed asks %s of every artifact its build seat builds; it is not answered since %s (%v), so the first registrations wait for it.", id, since[seat].UTC().Format(time.RFC3339), unanswered[seat]),
"indexer."+mcc.KindCallUnanswered)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("seed: the finding %s is not written: %v", id, err)
}
}
for seat := range answered {
if _, bad := unanswered[seat]; bad {
continue
}
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, mcc.KindCallUnanswered, "the seat answers steps"); err != nil {
ix.logf("seed: the finding %s.%s is not cleared: %v", seat, StepsVerb, err)
}
}
}
// namedPaths are the named paths the entry keeps for the artifact, by its name or its reference.
func namedPaths(e meshcatalog.CatalogEntry, name string) []string {
if e.Inputs == nil {
return nil
}
if p, ok := e.Inputs.Paths[name]; ok {
return p
}
return e.Inputs.Paths[ArtifactRefOf(e.Module, name)]
}
// StepsSincePrefix opens the keys of the state heard (Heard, A-80) that keep, by build seat, since when its steps
// has not answered (design 47 G A-66): a restart keeps the time, and the finding is written at the next seed attempt past
// five minutes.
const StepsSincePrefix = "steps-since."
// stepsUnansweredSince is the time the seat first failed to answer.
func (ix *Indexer) stepsUnansweredSince(ctx context.Context, seat string, now time.Time) time.Time {
return ix.unansweredSince(ctx, StepsSincePrefix+keys.EscapeName(seat), now)
}
// stepsAnswered forgets the time a seat that answers was unanswered since.
func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) {
ix.answered(ctx, StepsSincePrefix+keys.EscapeName(seat))
}
// unansweredSince is the time the call kept at key first failed to answer, kept in the indexer's state heard (the first
// failure writes now); where no state is kept it is held in memory.
func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Time) time.Time {
if ix.Heard == nil {
ix.mu.Lock()
defer ix.mu.Unlock()
if ix.stepsSince == nil {
ix.stepsSince = map[string]time.Time{}
}
if _, ok := ix.stepsSince[key]; !ok {
ix.stepsSince[key] = now
}
return ix.stepsSince[key]
}
if v, _, found, err := ix.Heard.Get(ctx, key); err == nil && found {
if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil {
return t
}
}
if err := ix.Heard.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil {
ix.logf("the time since %s is unanswered is not kept: %v", key, err)
}
return now
}
// answered forgets the time the call kept at key was unanswered since.
func (ix *Indexer) answered(ctx context.Context, key string) {
if ix.Heard == nil {
ix.mu.Lock()
delete(ix.stepsSince, key)
ix.mu.Unlock()
return
}
if _, rev, found, err := ix.Heard.Get(ctx, key); err == nil && found {
if err := ix.Heard.Delete(ctx, key, rev); err != nil {
ix.logf("the time since %s is unanswered is not forgotten: %v", key, err)
}
}
}