Live on novox the Go and Node build seats refused every seed steps call with 'the request names a module directory and no path': the trunk entries name no paths, and the seed sent the directory alone. Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
314 lines
12 KiB
Go
314 lines
12 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"path"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
|
)
|
|
|
|
// StepsVerb is the verb of each build seat the seed asks (design 47 G §704).
|
|
const StepsVerb = "steps"
|
|
|
|
// StepsCaller asks the build seat seat its verb steps (a kept call, ADR 0668 §1): the answer, or why none came.
|
|
type StepsCaller func(ctx context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error)
|
|
|
|
// ModuleManifest is the file name of a module's manifest in its directory.
|
|
const ModuleManifest = "module.json"
|
|
|
|
// errNoBuildSeat says no build seat's acting rule build matches an artifact's language or kind.
|
|
var errNoBuildSeat = errors.New("no seat builds the artifact")
|
|
|
|
// buildSeatOf is the seat whose acting rule `build` has `on` matching the artifact's language or kind (design 47 G §704,
|
|
// §203, §324), read from the seat list defs as BuildModulesOf reads it; none, or more than one, is said.
|
|
func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (string, error) {
|
|
var seats []string
|
|
for _, name := range slices.Sorted(maps.Keys(defs)) {
|
|
for _, r := range defs[name].Acts {
|
|
if r.Step != seatdef.StepBuild {
|
|
continue
|
|
}
|
|
if r.ActsOn(a.Language) || (r.On == seatdef.OnKindImage && a.Kind == "image") {
|
|
seats = append(seats, name)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
switch len(seats) {
|
|
case 0:
|
|
return "", fmt.Errorf("%w %s (language %q, kind %q)", errNoBuildSeat, a.Ref, a.Language, a.Kind)
|
|
case 1:
|
|
return seats[0], nil
|
|
}
|
|
return "", fmt.Errorf("the seats %v each build the artifact %s", seats, a.Ref)
|
|
}
|
|
|
|
// SeedModulesFrom reads the modules the newest trunk revision holds for the seed (design 47 G §704, §8): for each artifact of
|
|
// each module the seat list's build seat answers steps with {module, artifact, fingerprint, fetch} from the module's trunk
|
|
// entry, the fetch its repository, commit, directory and the artifact's named paths. A seat that does not answer leaves
|
|
// its artifact unanswered, which holds the seed; its call-unanswered finding (scope call, id <seat>.steps) is written once
|
|
// it is unanswered five minutes and cleared when it answers.
|
|
func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.Definition, steps StepsCaller) ([]SeedModule, error) {
|
|
if ix.Revisions == nil {
|
|
return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from")
|
|
}
|
|
rev, found, err := newestTrunk(ctx, ix.Revisions)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !found {
|
|
return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first")
|
|
}
|
|
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made
|
|
unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind
|
|
for _, e := range entries {
|
|
for _, a := range e.Artifacts {
|
|
if sdkstep.PackedOrFetched(a.Language, a.Kind) {
|
|
continue
|
|
}
|
|
seat, err := buildSeatOf(defs, a)
|
|
switch {
|
|
case errors.Is(err, errNoBuildSeat):
|
|
unbuilt[e.Module] = append(unbuilt[e.Module], unbuiltWords(a))
|
|
continue
|
|
case err != nil:
|
|
return nil, fmt.Errorf("%s: %w", e.Module, err)
|
|
}
|
|
seatOf[a.Ref] = seat
|
|
}
|
|
}
|
|
answered := map[string]bool{}
|
|
unanswered := map[string]error{}
|
|
var out []SeedModule
|
|
for _, e := range entries {
|
|
m := SeedModule{Module: e.Module, Origin: e.Repository, Path: path.Join(e.Path, ModuleManifest)}
|
|
if words, ok := unbuilt[e.Module]; ok {
|
|
m.LeftOut = "no build seat builds " + strings.Join(words, ", ")
|
|
out = append(out, m)
|
|
continue
|
|
}
|
|
for _, a := range e.Artifacts {
|
|
name := artifactName(a.Ref)
|
|
if sdkstep.PackedOrFetched(a.Language, a.Kind) {
|
|
// The builder packs or fetches it: no steps call, and its lint, unit and integration are bypass (A-324).
|
|
m.Artifacts = append(m.Artifacts, SeedArtifact{Name: name, Answered: true})
|
|
continue
|
|
}
|
|
seat := seatOf[a.Ref]
|
|
sa := SeedArtifact{Name: name, Seat: seat}
|
|
in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp,
|
|
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: maps.Clone(e.Packages)}}
|
|
// The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause
|
|
// 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest).
|
|
if paths := namedPaths(e, name); len(paths) > 0 {
|
|
in.Fetch.Dir, in.Fetch.Paths = e.Path, paths
|
|
}
|
|
if _, gone := unanswered[seat]; gone {
|
|
// the seat did not answer earlier in this seed: its later artifacts are not asked again (A-392)
|
|
m.Artifacts = append(m.Artifacts, sa)
|
|
continue
|
|
}
|
|
ans, err := steps(ctx, seat, in)
|
|
switch {
|
|
case errors.Is(err, ErrAskClosed): // the entry is no longer held: not said unanswered (A-190)
|
|
case err != nil:
|
|
unanswered[seat] = err
|
|
ix.logf("seed: the seat %s did not answer steps: %v; its later artifacts are not asked in this seed", seat, err)
|
|
case ans.Error != "":
|
|
unanswered[seat] = fmt.Errorf("%s", ans.Error)
|
|
ix.logf("seed: the seat %s did not answer steps: %s; its later artifacts are not asked in this seed", seat, ans.Error)
|
|
default:
|
|
sa.Answered = true
|
|
answered[seat] = true
|
|
for _, s := range ans.Steps {
|
|
sa.Steps = append(sa.Steps, s.Step)
|
|
}
|
|
}
|
|
m.Artifacts = append(m.Artifacts, sa)
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
ix.stepsFindings(ctx, answered, unanswered)
|
|
ix.leftOutFindings(ctx, entries, out)
|
|
return out, nil
|
|
}
|
|
|
|
// unbuiltWords names an artifact no build seat builds, with its language, else its kind.
|
|
func unbuiltWords(a meshcatalog.Artifact) string {
|
|
if a.Language != "" {
|
|
return fmt.Sprintf("%s (language %s)", a.Ref, a.Language)
|
|
}
|
|
return fmt.Sprintf("%s (kind %q)", a.Ref, a.Kind)
|
|
}
|
|
|
|
// leftOutFindings writes seed-left-out for each module the seed leaves out because no build seat builds an artifact of it,
|
|
// and clears it only for a module of which a build seat now builds every artifact, or which left the catalogue; a seed
|
|
// applied without the module clears nothing (design 47 G A-324, A-329).
|
|
func (ix *Indexer) leftOutFindings(ctx context.Context, entries []meshcatalog.CatalogEntry, mods []SeedModule) {
|
|
if ix.Findings == nil {
|
|
return
|
|
}
|
|
left := map[string]bool{}
|
|
for _, m := range mods {
|
|
if m.LeftOut == "" {
|
|
continue
|
|
}
|
|
left[keyID(m.Module)] = true
|
|
f := ix.observed(mcc.ScopeModule, m.Module, SeedLeftOutKind,
|
|
fmt.Sprintf("the seed leaves out %s: %s", m.Module, m.LeftOut),
|
|
"A module is left out of the first registrations",
|
|
fmt.Sprintf("The seed registers no module with an artifact no build seat builds, since a pipeline of it could not run that artifact's steps; %s is left out: %s.", m.Module, m.LeftOut),
|
|
"indexer."+SeedLeftOutKind)
|
|
if err := ix.Findings.Write(ctx, f); err != nil {
|
|
ix.logf("seed: the finding %s of %s is not written: %v", SeedLeftOutKind, m.Module, err)
|
|
}
|
|
}
|
|
inCatalogue := map[string]bool{}
|
|
for _, e := range entries {
|
|
inCatalogue[keyID(e.Module)] = true
|
|
}
|
|
for _, f := range ix.Findings.Held() {
|
|
if f.Kind != SeedLeftOutKind || f.Scope != mcc.ScopeModule || left[f.ID] {
|
|
continue
|
|
}
|
|
why := "a build seat builds every artifact of it"
|
|
if !inCatalogue[f.ID] {
|
|
why = "the module left the catalogue"
|
|
}
|
|
if err := ix.Findings.Clear(ctx, f.Scope, f.ID, f.Kind, why); err != nil {
|
|
ix.logf("seed: the finding %s of %s is not cleared: %v", SeedLeftOutKind, f.ID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// artifactName is the artifact's name in an artifact reference <module>:<artifact>.
|
|
func artifactName(ref string) string {
|
|
for i := len(ref) - 1; i >= 0; i-- {
|
|
if ref[i] == ':' {
|
|
return ref[i+1:]
|
|
}
|
|
}
|
|
return ref
|
|
}
|
|
|
|
// stepsFindings writes call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
|
|
func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, unanswered map[string]error) {
|
|
if ix.Findings == nil {
|
|
return
|
|
}
|
|
now := ix.now()
|
|
var raise []string
|
|
since := map[string]time.Time{}
|
|
for seat := range unanswered {
|
|
since[seat] = ix.stepsUnansweredSince(ctx, seat, now)
|
|
if now.Sub(since[seat]) >= mcc.CallUnansweredAfter {
|
|
raise = append(raise, seat)
|
|
}
|
|
}
|
|
for seat := range answered {
|
|
if _, bad := unanswered[seat]; !bad {
|
|
ix.stepsAnswered(ctx, seat)
|
|
}
|
|
}
|
|
for _, seat := range slices.Sorted(slices.Values(raise)) {
|
|
id := seat + "." + StepsVerb
|
|
f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered,
|
|
fmt.Sprintf("the call %s is unanswered since %s", id, since[seat].UTC().Format(time.RFC3339)),
|
|
"A call whose answer is kept is not answered",
|
|
fmt.Sprintf("The seed asks %s of every artifact its build seat builds; it is not answered since %s (%v), so the first registrations wait for it.", id, since[seat].UTC().Format(time.RFC3339), unanswered[seat]),
|
|
"indexer."+mcc.KindCallUnanswered)
|
|
if err := ix.Findings.Write(ctx, f); err != nil {
|
|
ix.logf("seed: the finding %s is not written: %v", id, err)
|
|
}
|
|
}
|
|
for seat := range answered {
|
|
if _, bad := unanswered[seat]; bad {
|
|
continue
|
|
}
|
|
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, mcc.KindCallUnanswered, "the seat answers steps"); err != nil {
|
|
ix.logf("seed: the finding %s.%s is not cleared: %v", seat, StepsVerb, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// namedPaths are the named paths the entry keeps for the artifact, by its name or its reference.
|
|
func namedPaths(e meshcatalog.CatalogEntry, name string) []string {
|
|
if e.Inputs == nil {
|
|
return nil
|
|
}
|
|
if p, ok := e.Inputs.Paths[name]; ok {
|
|
return p
|
|
}
|
|
return e.Inputs.Paths[ArtifactRefOf(e.Module, name)]
|
|
}
|
|
|
|
// StepsSincePrefix opens the keys of the state heard (Heard, A-80) that keep, by build seat, since when its steps
|
|
// has not answered (design 47 G A-66): a restart keeps the time, and the finding is written at the next seed attempt past
|
|
// five minutes.
|
|
const StepsSincePrefix = "steps-since."
|
|
|
|
// stepsUnansweredSince is the time the seat first failed to answer.
|
|
func (ix *Indexer) stepsUnansweredSince(ctx context.Context, seat string, now time.Time) time.Time {
|
|
return ix.unansweredSince(ctx, StepsSincePrefix+keys.EscapeName(seat), now)
|
|
}
|
|
|
|
// stepsAnswered forgets the time a seat that answers was unanswered since.
|
|
func (ix *Indexer) stepsAnswered(ctx context.Context, seat string) {
|
|
ix.answered(ctx, StepsSincePrefix+keys.EscapeName(seat))
|
|
}
|
|
|
|
// unansweredSince is the time the call kept at key first failed to answer, kept in the indexer's state heard (the first
|
|
// failure writes now); where no state is kept it is held in memory.
|
|
func (ix *Indexer) unansweredSince(ctx context.Context, key string, now time.Time) time.Time {
|
|
if ix.Heard == nil {
|
|
ix.mu.Lock()
|
|
defer ix.mu.Unlock()
|
|
if ix.stepsSince == nil {
|
|
ix.stepsSince = map[string]time.Time{}
|
|
}
|
|
if _, ok := ix.stepsSince[key]; !ok {
|
|
ix.stepsSince[key] = now
|
|
}
|
|
return ix.stepsSince[key]
|
|
}
|
|
if v, _, found, err := ix.Heard.Get(ctx, key); err == nil && found {
|
|
if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil {
|
|
return t
|
|
}
|
|
}
|
|
if err := ix.Heard.Create(ctx, key, []byte(now.UTC().Format(time.RFC3339Nano))); err != nil {
|
|
ix.logf("the time since %s is unanswered is not kept: %v", key, err)
|
|
}
|
|
return now
|
|
}
|
|
|
|
// answered forgets the time the call kept at key was unanswered since.
|
|
func (ix *Indexer) answered(ctx context.Context, key string) {
|
|
if ix.Heard == nil {
|
|
ix.mu.Lock()
|
|
delete(ix.stepsSince, key)
|
|
ix.mu.Unlock()
|
|
return
|
|
}
|
|
if _, rev, found, err := ix.Heard.Get(ctx, key); err == nil && found {
|
|
if err := ix.Heard.Delete(ctx, key, rev); err != nil {
|
|
ix.logf("the time since %s is unanswered is not forgotten: %v", key, err)
|
|
}
|
|
}
|
|
}
|