Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/versions_fill.go
T
jschoubben 5b13cc7ab1 IX24: the indexer's four kept calls are asked again only while their ask is open
messageVia, stepsVia, sourceVia and versionsVia go through CallKeptWhileOpen: open while the commit's take-in
is pending, the candidate is held, the branch is watched, the module is in the newest view. A call that ends
with its ask closed is index.ErrAskClosed, not an unanswered call.

Builds: design 47 G §351, §367, §460, §531; A-190

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 05:53:45 +01:00

211 lines
8.2 KiB
Go

package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The first fill of the versions (design 47 G §372): at the first start under G the state versions is filled once, as
// released, for each module from the highest published version over every artifact of the module that the builder's state
// versions held, read by the builder's tool builder_versions as a kept call; then versions-filled is said once, and the
// builder reserves nothing before it (§369).
const (
// BuilderVersionsCall is the tool the manifest names in its invokes (until that tool goes).
BuilderVersionsCall = "mesh-delivery-builder.builder_versions"
// VersionsFilledKey is the key of the state heard that says the fill is done, so it is not made again.
VersionsFilledKey = "versions-filled"
)
// versionsFillSince is the key of the state heard of the time builder_versions is first unanswered.
var versionsFillSince = "call-since." + keys.EscapeName(BuilderVersionsCall)
// BuilderVersion is one version of a name in the answer of builder_versions (its fields the fill reads).
type BuilderVersion struct {
Version string `json:"version"`
State string `json:"state"`
Digest string `json:"digest"`
Revision string `json:"revision"`
At time.Time `json:"at"`
}
// BuilderVersionsAnswer is what builder_versions answers for a name.
type BuilderVersionsAnswer struct {
Name string `json:"name"`
Versions []BuilderVersion `json:"versions"`
}
// VersionsAsker asks builder_versions for a name's versions (a kept call, ADR 0668 §1, §2).
type VersionsAsker func(ctx context.Context, module, name string) (BuilderVersionsAnswer, error)
// versionNameOf is the name the builder numbers an artifact under: its package's name in its ecosystem, else
// <module>/<artifact> (images, bundles and archives).
func versionNameOf(module string, a meshcatalog.Artifact) string {
if a.Package != nil && a.Package.Name != "" {
return a.Package.Name
}
return module + "/" + artifactName(a.Ref)
}
// FillVersions fills the state versions once from builder_versions and says versions-filled (§372). A fill already made
// is not made again; a builder_versions that does not answer holds the fill and, after five minutes, is the finding
// call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers.
func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
if ix.Versions == nil || ix.Heard == nil {
return errors.New("the state versions is not kept here")
}
if !ix.versionsFilling.CompareAndSwap(false, true) {
return nil // one fill runs at a time
}
defer ix.versionsFilling.Store(false)
if _, _, done, err := ix.Heard.Get(ctx, VersionsFilledKey); err != nil || done {
return err
}
if ix.Revisions == nil {
return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts")
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
if err != nil {
return err
}
if !found {
return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first")
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return err
}
type found1 struct {
v BuilderVersion
name string
ref string
}
best := map[string][]found1{} // by module: each artifact's highest published version
var unanswered error
for _, e := range entries {
var top string // the module's highest published version
per := []found1{}
for _, a := range e.Artifacts {
name := versionNameOf(e.Module, a)
ans, err := ask(ctx, e.Module, name)
if errors.Is(err, ErrAskClosed) {
continue // the module left the view: nothing waits on its versions (A-190)
}
if err != nil {
unanswered = err
continue
}
for _, v := range ans.Versions {
if _, ok := versionParts(v.Version); !ok || v.State != "published" {
continue
}
per = append(per, found1{v: v, name: name, ref: a.Ref})
if top == "" || compareVersions(v.Version, top) > 0 {
top = v.Version
}
}
}
for _, f := range per {
if f.v.Version == top {
best[e.Module] = append(best[e.Module], f)
}
}
}
if unanswered != nil {
ix.fillUnanswered(ctx, unanswered)
return fmt.Errorf("builder_versions is not answered: %w", unanswered)
}
ix.answered(ctx, versionsFillSince)
if ix.Findings != nil {
if err := ix.clearFinding(ctx, mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered, "builder_versions answers"); err != nil {
ix.logf("versions fill: the finding %s is not cleared: %v", BuilderVersionsCall, err)
}
}
now := ix.now().UTC()
filled := 0
built := map[string]meshcatalog.VersionEntry{}
claimed := map[string]time.Time{}
trunk := map[string]meshcatalog.CatalogEntry{}
for _, e := range entries {
trunk[e.Module] = e
}
for module, fs := range best {
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: module, Version: fs[0].v.Version, At: &now}
for _, f := range fs {
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: f.ref, Digest: f.v.Digest})
if n, err := strconv.ParseInt(f.v.Revision, 10, 64); err == nil && n > v.Source {
v.Source = n
}
if f.v.At.After(claimed[module]) {
claimed[module] = f.v.At
}
}
built[module] = v
}
// The edge entries (§377), each marked filled: where the stream running or the placement cannot be read the entries of kinds 2, 4
// and 6 read "not yet delivered", as at a release, and the first delivery writes them.
var running map[string]map[string]nodeengine.Running
if ix.RunningWhole != nil {
if running, err = ix.RunningWhole(ctx); err != nil {
ix.logf("versions fill: the stream running is not read whole: %v", err)
running = nil
}
}
answers := map[string]BuilderVersionsAnswer{}
for _, module := range slices.Sorted(maps.Keys(built)) {
v := built[module]
v.Edges = ix.filledEdges(ctx, trunk[module], claimed[module], ask, trunk, built, running, answers)
built[module] = v
}
for _, module := range slices.Sorted(maps.Keys(built)) {
v := built[module]
data, err := json.Marshal(v)
if err != nil {
return err
}
if err := ix.Versions.Create(ctx, meshcatalog.VersionsByModuleVersionKey(module, v.Version), data); err != nil && !errors.Is(err, ErrConflict) {
return fmt.Errorf("the version %s %s is not kept: %w", module, v.Version, err)
}
ix.keepFilledFacts(ctx, v, trunk[module], built, now)
filled++
}
if ix.SayVersionsFilled != nil {
ev := meshcatalog.VersionsFilledEvent{V: int64(keys.Version), At: now, Modules: int64(filled)}
if err := ix.SayVersionsFilled(ctx, ev, meshcatalog.EventVersionsFilled); err != nil {
return fmt.Errorf("versions-filled is not said: %w", err)
}
}
if err := ix.Heard.Create(ctx, VersionsFilledKey, []byte(now.Format(time.RFC3339Nano))); err != nil && !errors.Is(err, ErrConflict) {
return err
}
ix.logf("versions: filled for %d modules from builder_versions, versions-filled said", filled)
return nil
}
// fillUnanswered writes call-unanswered once builder_versions is unanswered five minutes (the time kept in the own state).
func (ix *Indexer) fillUnanswered(ctx context.Context, cause error) {
now := ix.now()
since := ix.unansweredSince(ctx, versionsFillSince, now)
if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter {
return
}
f := ix.observed(mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered,
fmt.Sprintf("the call %s is unanswered since %s", BuilderVersionsCall, since.UTC().Format(time.RFC3339)),
"A call whose answer is kept is not answered",
fmt.Sprintf("The first fill of the catalog's versions asks %s of every artifact; it is not answered since %s (%v), so no version is released and the builder reserves nothing.", BuilderVersionsCall, since.UTC().Format(time.RFC3339), cause),
"indexer."+mcc.KindCallUnanswered)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("versions fill: the finding %s is not written: %v", BuilderVersionsCall, err)
}
}